In [1]:
%matplotlib inline
%config InlineBackend.figure_format='retina'
from IPython.display import Image, HTML

import random
import math
import pysolar
import datetime as dt
import pytz
import matplotlib.dates as mpd
import scipy
import numexpr as ne

from datacharm import *
from datacharm.dataplots import tableau20, ch_color

from enerpi.base import timeit
from enerpi.api import enerpi_data_catalog
from enerpi.enerplot import plot_tile_last_24h, plot_power_consumption_hourly
from enerpi.process import separa_ldr_artificial_natural, get_solar_day
from enerpi.radsolar import ajuste_ldr_con_barrido_irrad_inc, local_maxima_minima


LAT = 38.631463
LONG = -0.866402
ELEV_M = 500
TZ = pytz.timezone('Europe/Madrid')
FS = (16, 10)


def _tslim(ax, h0=12, hf=22, m0=0, mf=0):
    xlim = [mpd.num2date(x, tz=TZ) for x in ax.get_xlim()]
    new = [mpd.date2num(d.replace(hour=h, minute=m)) for d, h, m in zip(xlim, [h0, hf], [m0, mf])]
    ax.set_xlim(new)
    return ax


def set_sun_times(df, tz=TZ, lat_deg=LAT, long_deg=LONG, offset='5min'):
    if df.index[-1].date() == df.index[0].date():
        # Same day:
        ts_day = pd.Timestamp(df.index[0].date(), tz=tz)
        sunrise, sunset = pysolar.util.get_sunrise_sunset(lat_deg, long_deg, ts_day + pd.Timedelta('12h'))
        delta = pd.Timedelta(offset)
        sunrise, sunset = [sunrise - delta, sunset + delta]
        df['sun'] = False
        df.loc[sunrise:sunset, 'sun'] = True
        return df
    else:
        print_err('IMPLEMENTAR!')
        assert()


sns.set_style('whitegrid')        

# Catálogo y lectura de todos los datos.
cat = enerpi_data_catalog()
data, data_s = cat.get_all_data()
print_info(data_s.tail())

LDR = pd.DataFrame(data.ldr).tz_localize(TZ)
print_cyan(LDR.describe().T.astype(int))
LDR.hist(bins=(LDR.max() - LDR.min() + 1).values[0] // 5, log=True, figsize=(18, 8))

Image('/Users/uge/Desktop/LDR_analysis/LDR_análisis_día_2016_09_12.png')


 ==> Librerías, clases y métodos cargados:
 +++ "dt"                           +++ "os" 
 +++ "json" v:2.0.9                 +++ "pd" (pandas) v:0.18.1
 +++ "locale"                       +++ "plt" 
 +++ "math"                         +++ "re" v:2.2.1
 +++ "np" (numpy) v:1.11.1          +++ "sns" (seaborn) v:0.7.0
 +++ "sys" 
 ** "Colormap", "Line2D", "Normalize", "OrderedDict", "PathPatch", "namedtuple", "time"
 ==> Pretty printing funcs:
print_blue, print_bold, print_boldu, print_cyan, print_err, print_green, print_grey, print_greyb, print_info, print_infob, print_magenta, print_ok, print_red, print_redb, print_secc, print_tree_dict, print_warn, print_white, print_yellow, print_yellowb, printcolor
If you are in a jupyter notebook, insert this:
%matplotlib inline
%config InlineBackend.figure_format='retina'
If you are working with GEO data, insert this:
import geopandas as gpd
import shapely.geometry as sg
import cartopy.crs as ccrs
/Users/uge/anaconda/envs/py35/lib/python3.5/site-packages/matplotlib/__init__.py:1350: UserWarning:  This call to matplotlib.use() has no effect
because the backend has already been chosen;
matplotlib.use() must be called *before* pylab, matplotlib.pyplot,
or matplotlib.backends is imported for the first time.

  warnings.warn(_use_error_msg)
***TIMEIT get_all_data TOOK: 0.984 s
                          kWh     t_ref  n_jump  n_exec   p_max  p_mean  p_min
ts                                                                            
2016-09-14 09:00:00  0.280975  0.999865       0       0   865.0   281.0  185.0
2016-09-14 10:00:00  0.348163  1.000116       0       0   422.0   348.0  273.0
2016-09-14 11:00:00  0.347181  1.000035       0       0   883.0   347.0  277.0
2016-09-14 12:00:00  0.312149  0.999837       0       0   391.0   312.0  269.0
2016-09-14 13:00:00  0.144251  0.331557       0       0  2564.0   435.0  284.0
       count  mean  std  min  25%  50%  75%  max
ldr  2725278   251  234   11   33  162  478  748
Out[1]:

Análisis de 1 día --> '2016-09-12'


In [2]:
#day = '2016-08-27'
day = '2016-09-12'

data_raw = LDR.loc[day:day]
print_info(data_raw.head(3))

data, data_simple = separa_ldr_artificial_natural(data_raw, resample_inicial='2s', delta_roll_threshold=100)
solar = get_solar_day(data)

cols_p = ['ldr', 'delta_roll','artif_level_max', 'altitud','irradiacion_cs']
solar[cols_p].iloc[::5].plot(figsize=FS)


                                  ldr
ts                                   
2016-09-12 00:00:00.377032+02:00   40
2016-09-12 00:00:01.377852+02:00   40
2016-09-12 00:00:02.387928+02:00   41
                             ldr  delta_roll  ch_state
ts                                                    
2016-09-12 09:48:14+02:00  496.5      -148.0        -1
2016-09-12 10:02:22+02:00  470.0       154.5         1
2016-09-12 15:27:54+02:00  660.0       108.0         1
2016-09-12 15:29:26+02:00  606.5      -102.0        -1
2016-09-12 18:39:26+02:00  470.5       240.5         1
2016-09-12 19:14:58+02:00  613.5      -102.5        -1
2016-09-12 19:27:46+02:00  335.0       292.5         1
2016-09-12 19:28:20+02:00  611.0      -162.5        -1
2016-09-12 19:28:34+02:00  317.0       344.0         1
2016-09-12 19:32:30+02:00  522.5      -277.5        -1
2016-09-12 19:33:34+02:00  237.5      -216.0        -1
2016-09-12 19:46:20+02:00  102.0       137.5         1
2016-09-12 19:56:36+02:00  262.0       122.5         1
2016-09-12 22:23:54+02:00  205.0      -114.5        -1
2016-09-12 22:24:32+02:00   64.0       304.5         1
Encendido L=734, SEGS=19624
Encendido L=698, SEGS=92
Encendido L=625, SEGS=2132
Encendido L=611, SEGS=34
Encendido L=623, SEGS=236
Encendido L=331, SEGS=9454
Encendido L=322, SEGS=8838
separa_ldr_artificial_natural TOOK: 0.771 s
/Users/uge/anaconda/envs/py35/lib/python3.5/site-packages/pysolar/time.py:105: UserWarning: I don't know about leap seconds after 2015
  (leap_seconds_base_year + len(leap_seconds_adjustments) - 1)
get_solar_day TOOK: 0.465 s
Out[2]:
<matplotlib.axes._subplots.AxesSubplot at 0x112ae4cc0>

In [8]:
from scipy.signal import medfilt


#panel = pd.read_hdf('/Users/uge/Desktop/LDR_analysis/POIS.h5', 'POIS')
#panel.loc['2016-08-28']
#ts = pd.Timestamp('2016-08-28')

#df = panel.loc['2016-08-28']
#df['day'] = ts.date()
#from enerpi.radsolar import identifica_pois_ldr

@timeit('identifica_pois_LDR', verbose=True)
def _identifica_pois_ldr(data_raw):
    def _es_max_local(x):
        mid = len(x) // 2
        if (np.argmax(x) == mid) and (x[mid] > 10):
            return True
        return False

    def _pendiente(x):
        return (x[-1] - x[0]) / len(x)

    def _pendiente_ini(x):
        if len(x) > 100:
            return _pendiente(x.values[1:101])
        return _pendiente(x.values)

    def _pendiente_fin(x):
        if len(x) > 100:
            return _pendiente(x.values[-100:-1])
        return _pendiente(x.values)

    def _pendiente_last_gen(x):
        limit = min(len(x), 600)
        if limit > 1:
            return _pendiente(x.values[-limit:-1])
        print_err(x)
        return np.nan

    def _get_alt(d):
        alt = pysolar.solar.get_altitude_fast(LAT, LONG, d)
        if alt > 0:
            return alt
        return 0

    # Resampling 1s + sun
    data_homog = data_raw.resample('1s').mean().interpolate().dropna()
    data_homog = set_sun_times(data_homog, tz=TZ, lat_deg=LAT, long_deg=LONG, offset='5min')
    subset_pysolar = data_homog.iloc[::120].index
    data_homog = data_homog.join(pd.Series(subset_pysolar, index=subset_pysolar, name='altitud'
                                           ).apply(_get_alt)).interpolate()

    # Median filter
    data_homog['median_filter'] = medfilt(data_homog.ldr, kernel_size=301)

    # BUSCA POI's
    data_homog['r_disp'] = data_homog.median_filter.rolling(5, center=True).apply(lambda x: x.max() - x.min()).fillna(0)
    data_homog['poi'] = data_homog['r_disp'].rolling(5).apply(_es_max_local).fillna(False).astype(bool)

    # Construye intervalos entre POI's
    data_homog['intervalo'] = data_homog['poi'].cumsum()

    # DATOS POIS:
    gb_poi = data_homog.tz_localize(None).reset_index().groupby('intervalo')
    pois = pd.DataFrame(pd.concat([gb_poi.sun.apply(lambda x: np.sum(x) / len(x)).rename('fr_sun'),
                                   gb_poi.ldr.count().rename('seconds'),
                                   gb_poi.altitud.mean(),
                                   gb_poi.median_filter.apply(lambda x: (x.values[-1] - x.values[0]) / len(x)
                                                              ).rename('pendiente_tot'),
                                   gb_poi.median_filter.apply(lambda x: np.round(np.median(x.values[1:10]))
                                                              ).rename('inicio'),
                                   gb_poi.median_filter.apply(lambda x: np.round(np.median(x.values[-10:-1]))
                                                              ).rename('fin'),
                                   gb_poi.median_filter.apply(_pendiente_ini).rename('pendiente_i'),
                                   gb_poi.median_filter.apply(_pendiente_fin).rename('pendiente_f'),
                                   gb_poi.median_filter.apply(_pendiente_last_gen).rename('pendiente_last_mf'),
                                   gb_poi.ts.first().rename('ts_ini'),
                                   gb_poi.ts.last().rename('ts_fin')], axis=1))
    pois['step'] = (pois['inicio'] - pois['fin'].shift())  # .fillna(0)
    print_ok(pois.head(10))
    return data_homog, pois


df = LDR.loc['2016-08-15']

data_homog, pois = _identifica_pois_ldr(df)
pois


ERROR: 86399    17.0
Name: 27, dtype: float64
             fr_sun  seconds    altitud  pendiente_tot  inicio    fin  pendiente_i  pendiente_f  pendiente_last_mf              ts_ini              ts_fin   step
intervalo                                                                                                                                                        
0          0.000000        5   0.000000       1.600000    26.0   22.0     1.600000     1.600000           2.000000 2016-08-15 00:00:00 2016-08-15 00:00:04    NaN
1          0.000000     1145   0.000000       0.017467    35.0   42.0     0.090000     0.111111           0.018364 2016-08-15 00:00:05 2016-08-15 00:19:09   13.0
2          0.241996    32141   2.217707       0.006254    58.0  246.0     0.090000     0.111111           0.045075 2016-08-15 00:19:10 2016-08-15 09:14:50   16.0
3          1.000000     3523  27.178003       0.080897   263.0  473.0     0.230000     0.828283           0.181970 2016-08-15 09:14:51 2016-08-15 10:13:33   17.0
4          1.000000        4  32.882000       4.000000   557.0  549.0     4.000000     4.000000           4.666667 2016-08-15 10:13:34 2016-08-15 10:13:37   84.0
5          1.000000       78  33.013663       0.525641   564.0  588.0     0.525641     0.525641           0.415584 2016-08-15 10:13:38 2016-08-15 10:14:55   15.0
6          1.000000     9389  47.284524      -0.012674   609.0  555.0     0.170000    -0.555556          -0.118531 2016-08-15 10:14:56 2016-08-15 12:51:24   21.0
7          1.000000      529  60.256920      -0.240076   439.0  411.0    -0.060000    -0.575758          -0.159091 2016-08-15 12:51:25 2016-08-15 13:00:13 -116.0
8          1.000000     1821  62.265022       0.101593   299.0  267.0    -0.250000     1.656566           0.278798 2016-08-15 13:00:14 2016-08-15 13:30:34 -112.0
9          1.000000      697  63.896077       0.077475   534.0  560.0     0.350000     0.000000           0.023372 2016-08-15 13:30:35 2016-08-15 13:42:11  267.0
identifica_pois_LDR TOOK: 1.351 s
/Users/uge/anaconda/envs/py35/lib/python3.5/site-packages/numpy/core/_methods.py:59: RuntimeWarning: Mean of empty slice.
  warnings.warn("Mean of empty slice.", RuntimeWarning)
Out[8]:
fr_sun seconds altitud pendiente_tot inicio fin pendiente_i pendiente_f pendiente_last_mf ts_ini ts_fin step
intervalo
0 0.000000 5 0.000000 1.600000 26.0 22.0 1.600000 1.600000 2.000000 2016-08-15 00:00:00 2016-08-15 00:00:04 NaN
1 0.000000 1145 0.000000 0.017467 35.0 42.0 0.090000 0.111111 0.018364 2016-08-15 00:00:05 2016-08-15 00:19:09 13.0
2 0.241996 32141 2.217707 0.006254 58.0 246.0 0.090000 0.111111 0.045075 2016-08-15 00:19:10 2016-08-15 09:14:50 16.0
3 1.000000 3523 27.178003 0.080897 263.0 473.0 0.230000 0.828283 0.181970 2016-08-15 09:14:51 2016-08-15 10:13:33 17.0
4 1.000000 4 32.882000 4.000000 557.0 549.0 4.000000 4.000000 4.666667 2016-08-15 10:13:34 2016-08-15 10:13:37 84.0
5 1.000000 78 33.013663 0.525641 564.0 588.0 0.525641 0.525641 0.415584 2016-08-15 10:13:38 2016-08-15 10:14:55 15.0
6 1.000000 9389 47.284524 -0.012674 609.0 555.0 0.170000 -0.555556 -0.118531 2016-08-15 10:14:56 2016-08-15 12:51:24 21.0
7 1.000000 529 60.256920 -0.240076 439.0 411.0 -0.060000 -0.575758 -0.159091 2016-08-15 12:51:25 2016-08-15 13:00:13 -116.0
8 1.000000 1821 62.265022 0.101593 299.0 267.0 -0.250000 1.656566 0.278798 2016-08-15 13:00:14 2016-08-15 13:30:34 -112.0
9 1.000000 697 63.896077 0.077475 534.0 560.0 0.350000 0.000000 0.023372 2016-08-15 13:30:35 2016-08-15 13:42:11 267.0
10 1.000000 6196 63.367927 0.010329 538.0 612.0 0.000000 -0.292929 -0.051753 2016-08-15 13:42:12 2016-08-15 15:25:27 -22.0
11 1.000000 3623 55.149733 -0.011041 592.0 574.0 -0.320000 -0.464646 -0.118531 2016-08-15 15:25:28 2016-08-15 16:25:50 -20.0
12 1.000000 7 50.385091 -2.714286 544.0 546.0 -2.714286 -2.714286 -2.500000 2016-08-15 16:25:51 2016-08-15 16:25:57 -30.0
13 1.000000 163 50.144536 0.049080 528.0 528.0 -0.010000 0.111111 0.043210 2016-08-15 16:25:58 2016-08-15 16:28:40 -18.0
14 1.000000 7839 37.941256 -0.016966 551.0 424.0 0.120000 -0.101010 -0.016694 2016-08-15 16:28:41 2016-08-15 18:39:19 23.0
15 1.000000 2824 20.870162 -0.028683 406.0 315.0 -0.120000 0.101010 -0.016694 2016-08-15 18:39:20 2016-08-15 19:26:23 -18.0
16 1.000000 1644 13.617632 -0.096107 332.0 319.0 0.040000 -0.949495 -0.131886 2016-08-15 19:26:24 2016-08-15 19:53:47 17.0
17 1.000000 1963 7.819738 -0.030056 135.0 93.0 -0.060000 -0.363636 -0.080134 2016-08-15 19:53:48 2016-08-15 20:26:30 -184.0
18 0.850373 3081 1.137466 0.058423 72.0 51.0 -0.030000 1.919192 0.318865 2016-08-15 20:26:31 2016-08-15 21:17:51 -21.0
19 0.000000 348 0.000000 0.060345 261.0 265.0 0.040000 0.101010 0.051873 2016-08-15 21:17:52 2016-08-15 21:23:39 210.0
20 0.000000 751 0.000000 0.000000 279.0 265.0 0.040000 0.151515 -0.006678 2016-08-15 21:23:40 2016-08-15 21:36:10 14.0
21 0.000000 2596 0.000000 -0.001926 284.0 288.0 0.030000 -0.101010 -0.001669 2016-08-15 21:36:11 2016-08-15 22:19:26 19.0
22 0.000000 13 0.000000 -3.846154 272.0 271.0 -3.846154 -3.846154 -2.500000 2016-08-15 22:19:27 2016-08-15 22:19:39 -16.0
23 0.000000 10 0.000000 -11.400000 193.0 208.0 -11.400000 -11.400000 -6.333333 2016-08-15 22:19:40 2016-08-15 22:19:49 -78.0
24 0.000000 4910 0.000000 -0.004684 74.0 40.0 -0.020000 0.030303 0.006678 2016-08-15 22:19:50 2016-08-15 23:41:39 -134.0
25 0.000000 543 0.000000 -0.001842 62.0 61.0 0.190000 -0.171717 0.001845 2016-08-15 23:41:40 2016-08-15 23:50:42 22.0
26 0.000000 556 0.000000 -0.043165 37.0 30.0 -0.020000 -0.101010 -0.032432 2016-08-15 23:50:43 2016-08-15 23:59:58 -24.0
27 0.000000 1 0.000000 0.000000 NaN NaN 0.000000 0.000000 NaN 2016-08-15 23:59:59 2016-08-15 23:59:59 NaN

In [10]:
data_homog.ldr.plot()


Out[10]:
<matplotlib.axes._subplots.AxesSubplot at 0x1129aed30>

In [26]:
def _plot_bt_medfilt(data_homog, t0='0:00', tf='1:00', ks_1=301, ks_2=75):
    df = data_homog.ldr.between_time(t0, tf)
    ax = df.plot(figsize=FS, color=tableau20[6], lw=.75, alpha=.7)
    pd.Series(medfilt(df, kernel_size=ks_1), index=df.index, 
              name='Medfilt_301').plot(ax=ax, color=tableau20[4], lw=4, alpha=.5)
    pd.Series(medfilt(df, kernel_size=ks_2), index=df.index, 
              name='Medfilt_301').plot(ax=ax, color=tableau20[2], lw=3, alpha=.6)
    return ax


_plot_bt_medfilt(data_homog, t0='0:00', tf='1:00', ks_1=301, ks_2=75)
plt.show()
_plot_bt_medfilt(data_homog, t0='15:00', tf='18:00', ks_1=301, ks_2=75)
plt.show()
_plot_bt_medfilt(data_homog, t0='19:00', tf='23:59:59', ks_1=301, ks_2=75);



In [76]:
@timeit('SUNTIMES', verbose=True)
def _suntimes(df):
    tt = pd.DatetimeIndex(df.index.date, tz=TZ)
    delta = pd.Timedelta('10min')
    s_sun = pd.Series([False] * len(df), index=df.index, name='sun')
    
    for ts_day in pd.DatetimeIndex(set(df.index.date), tz=TZ):
        sunrise, sunset = pysolar.util.get_sunrise_sunset(LAT, LONG, ts_day + pd.Timedelta('12h'))
        sunrise, sunset = [sunrise - delta, sunset + delta]
        s_sun.loc[sunrise:sunset] = True
        

data_ss = LDR.resample('5s').max()
df = _suntimes(data_ss)


SUNTIMES TOOK: 4.144 s

In [203]:
#pois = pd.read_hdf('/Users/uge/Desktop/LDR_analysis/POIS_debug.h5', 'POIS').iloc[:-1].fillna(0)
#print_info(pois.describe())

sns.pairplot(pois_labeled.drop(['ts_ini', 'is_poi_to_label'], axis=1))
#pois.drop(['ts_ini', 'ts_fin'], axis=1)


Out[203]:
<seaborn.axisgrid.PairGrid at 0x11ac5e550>

In [250]:
from datacharm.datafuncs import trimedia_TM, kmeans_1d, kmeans_md, clustering_pca_2d, silhouette_analysis, KMeans, PCA

X = pois_labeled.drop(['ts_ini', 'is_poi_to_label', 'intervalo'], axis=1).astype(float).values

#centers_km_c, labels_km_c, X_km_c, km_c, df_cl_c = kmeans_1d(pois_labeled, 'step', 7) #, func_preprocess=np.sqrt)
#silhouette_analysis(centers_km_c, labels_km_c, X_km_c)

centers_km_c, labels_km_c, X_km_c, km_c = kmeans_md(pois_labeled, ['step', 'altitud', 'ldr_median', 'pendiente_tot'], 4, sort_labels=False)
silhouette_analysis(centers_km_c, labels_km_c, X_km_c)


   index           0          1           2         3  items
0      0  -33.084034   7.126050  137.726891  0.619819    119
1      1  514.072727  12.963636  573.954545 -6.458695     55
2      2   51.701754  42.000000  508.333333 -0.736679    114
3      3 -492.500000   6.000000   42.392857  1.892375     56
kmeans_md TOOK: 0.013 s
For n_clusters = 4 The average silhouette_score is : 0.560980624184
silhouette_analysis TOOK: 0.929 s

In [247]:
# pois.columns
'''['intervalo', 'fr_sun', 'ini_sun', 'fin_sun', 'seconds', 'altitud', 'ldr_mean', 'ldr_median', 'pendiente_tot',
       'inicio', 'fin', 'pendiente_i', 'pendiente_f', 'pendiente_last_mf', 'ts_ini', 'step', 'is_poi_to_label',
       'solo_natural', 'artificial_princ']'''

cols_clustering = ['fr_sun', 'ini_sun', 'fin_sun', 'seconds', 'altitud', 'ldr_mean', 'ldr_median', 'pendiente_tot',
                   'inicio', 'fin', 'pendiente_i', 'pendiente_f', 'pendiente_last_mf', 'step']
centers, labels, X, km = kmeans_md(pois_labeled, cols_clustering, k, sort_labels=False)


   index         0         1         2              3          4           5           6         7           8  \
0      0  0.529195  0.546218  0.516807     763.991597  18.537815  331.081821  332.672269 -1.116742  333.957983   
1      1  0.303417  0.083333  1.000000   32883.750000   6.000000   98.052628   75.416667  0.041595   62.833333   
2      2  0.585818  0.000000  0.000000   83345.000000  19.000000   -0.995021   -1.000000  0.001800   -1.000000   
3      3  0.673182  0.700000  0.666667    9676.333333  27.800000  358.891134  363.833333  0.024381  357.433333   
4      4  0.707143  0.625000  0.875000   22018.750000  28.625000  336.658623  340.875000  0.010654  319.750000   
5      5  0.524477  0.500000  1.000000   64772.500000  16.500000  208.967302  164.000000  0.009537  254.000000   
6      6  0.635498  1.000000  1.000000  101865.000000  24.000000  229.911083  199.000000 -0.014824  510.000000   
7      7  0.610314  0.673913  0.543478    4525.434783  21.000000  284.197290  285.847826  0.104124  293.717391   
8      8  0.525052  0.333333  0.833333   42303.333333  16.000000  217.274144  147.166667  0.032488  122.166667   

            9        10        11        12          13  items  
0  332.546218 -1.193481 -1.043723 -0.527755   57.878151    238  
1  261.000000  0.032500  0.824916  0.260991 -341.500000     12  
2   -1.000000  0.000000  0.272727  0.045075  -31.000000      1  
3  333.233333  0.038000  0.192929 -0.000445  -74.900000     30  
4  311.625000  0.115000  0.165404  0.060309  -77.500000      8  
5  344.500000  0.005000  0.010101 -0.023372   11.500000      2  
6  480.000000 -0.040000 -1.353535 -0.195326   44.000000      1  
7  272.695652 -0.023043  0.781730  0.113916  -80.934783     46  
8  332.166667  0.036667  0.786195  0.088203  -78.666667      6  
kmeans_md TOOK: 0.024 s

In [248]:
len(pois), len(pois_labeled), len(pois_to_calc)


Out[248]:
(813, 344, 469)

In [251]:
pois_km = pois.drop(['ts_ini', 'intervalo'], axis=1).copy()

#pois_km['km_group'] = labels
pois_km.head()


Out[251]:
fr_sun ini_sun fin_sun seconds altitud ldr_mean ldr_median pendiente_tot inicio fin pendiente_i pendiente_f pendiente_last_mf step is_poi_to_label solo_natural artificial_princ
ts_fin
2016-08-12 10:46:45 1.0 1.0 1.0 25.0 40.0 0.000000 0.0 0.000000 0.0 0.0 0.000000 0.000000 0.000000 NaN False NaN NaN
2016-08-12 11:36:50 1.0 1.0 1.0 3005.0 44.0 648.910150 649.0 -0.051581 656.0 641.0 -0.010000 -0.171717 -0.048414 656.0 True False False
2016-08-12 11:42:55 1.0 1.0 1.0 365.0 49.0 627.041096 634.0 -1.780822 610.0 636.0 -1.780822 -1.780822 -0.944444 -31.0 True False False
2016-08-12 11:43:45 1.0 1.0 1.0 50.0 50.0 467.600000 438.0 13.100000 438.0 438.0 13.100000 13.100000 10.000000 -198.0 False NaN NaN
2016-08-12 11:59:50 1.0 1.0 1.0 965.0 51.0 630.113990 630.0 0.020725 636.0 632.0 -0.030000 0.040404 0.104167 198.0 True False False

In [291]:
#pois_km[pois_km.artificial_princ.notnull() & pois_km.artificial_princ]
#pois.drop(['ts_ini', 'artificial_princ', 'solo_natural', 'is_poi_to_label', 'intervalo'], axis=1)

In [293]:
#saltos = pois.join((pois['fin'] - pois['inicio'].shift(-1)).rename('step_salto'))
#saltos = saltos.join((saltos['pendiente_f'] - saltos['pendiente_i'].shift(-1)).rename('pendiente_salto'))
#saltos = saltos.iloc[1:-1][['seconds', 'fin_sun', 'fin', 'solo_natural', 'artificial_princ', 'step_salto', 'pendiente_salto']]
#saltos_labeled = saltos[saltos.solo_natural.notnull()]
#saltos_to_calc = saltos[saltos.solo_natural.isnull()]
#ts = saltos_to_calc.index.time[9]
#saltos.seconds.hist(bins=1000, figsize=FS, lw=0, log=True)
#(saltos.seconds).value_counts().sort_index()

In [262]:
k = 9
km_pca = clustering_pca_2d(X, k, whiten=True, ax=None)

X_2 = pois.drop(['ts_ini', 'artificial_princ', 'solo_natural', 'is_poi_to_label', 'intervalo'], axis=1).astype(float).iloc[1:].values
km_pca_2 = clustering_pca_2d(X_2, k, whiten=True, ax=None)


explained variance ratio (first two components): [ 0.99789111  0.00178308]
clustering_pca_lda TOOK: 0.977 s
explained variance ratio (first two components): [ 0.99734433  0.00220023]
clustering_pca_lda TOOK: 0.975 s

In [316]:
signal.general_gaussian?

In [346]:
from scipy import signal


#pd.set_option('display.width', 120)
path_st = '/Users/uge/Desktop/LDR_analysis/POIS_ident_1s.h5'
# data_homog, data_rs, pois = genera_pois_de_all_data_LDR(path_st)

pois = pd.read_hdf(path_st, 'POIS')
data_homog = pd.read_hdf(path_st, 'data_homog')
data_rs = pd.read_hdf(path_st, 'data_rs')
#pois_labeled = pois[pois.solo_natural.notnull()]
#pois_to_calc = pois[pois.solo_natural.isnull()]
#print(len(pois), len(pois_labeled), len(pois_to_calc))
print(len(pois), pois.columns)

data = data_rs.join(data_homog.ldr.rename('raw_ldr')).drop('sun', axis=1)
#data.reset_index().iloc[50:100]

window = signal.general_gaussian(51, p=0.5, sig=20)
filtered = signal.fftconvolve(window, data.raw_ldr)
filtered = (np.average(data.raw_ldr) / np.average(filtered)) * filtered
filtered = np.roll(filtered, -25)
print(len(filtered), len(data), window)
data['filter_g'] = filtered[:-50]
data.iloc[6000:8000].drop(['intervalo', 'altitud'], axis=1).plot(figsize=FS, lw=2, alpha=.7)


932 Index(['fr_sun', 'seconds', 'alt_sun_ini', 'alt_sun_fin', 'altitud', 'ldr_mean', 'ldr_min', 'ldr_max', 'ldr_median',
       'ldr_std', 'pendiente_tot', 'inicio', 'fin', 'pendiente_i', 'pendiente_f', 'pendiente_last_mf', 'ts_ini',
       'ts_fin', 'step', 'pendiente_salto', 'is_poi_to_label'],
      dtype='object')
2860458 2860408 [ 0.53526143  0.54881164  0.56270487  0.57694981  0.59155536  0.60653066
  0.62188506  0.63762815  0.65376979  0.67032005  0.68728928  0.70468809
  0.72252735  0.74081822  0.75957212  0.77880078  0.79851622  0.81873075
  0.83945702  0.86070798  0.8824969   0.90483742  0.92774349  0.95122942
  0.97530991  1.          0.97530991  0.95122942  0.92774349  0.90483742
  0.8824969   0.86070798  0.83945702  0.81873075  0.79851622  0.77880078
  0.75957212  0.74081822  0.72252735  0.70468809  0.68728928  0.67032005
  0.65376979  0.63762815  0.62188506  0.60653066  0.59155536  0.57694981
  0.56270487  0.54881164  0.53526143]
Out[346]:
<matplotlib.axes._subplots.AxesSubplot at 0x175d43c50>

In [375]:
import statsmodels.api as sm

#sm.tsa.ARIMA?

day = data.loc['2016-09-04']
print(day.shape)
#data.median_filter.shape
day.plot()
#data.median_filter.rolling(10).apply(np.ptp)
#np.ptp?


(86400, 8)
Out[375]:
<matplotlib.axes._subplots.AxesSubplot at 0x1706ce518>

In [362]:
data['mf_dmax'] = data['median_filter'].rolling(10).max() - data['median_filter']
data['mf_dmin'] = data['median_filter'] - data['median_filter'].rolling(10).min()
data['mf_dif'] = data['mf_dmax'] - data['mf_dmin']
#data['median_filter'].rolling(10).max() - data['median_filter'].rolling(10).min()

ax = data[['mf_dif', 'median_filter']].iloc[5000:7000].plot(figsize=FS, lw=2, alpha=.6) #, color=tableau20[0])
#data['median_filter'].rolling(10).min().iloc[6000:7000].plot(ax=ax, color=tableau20[2])
#data['median_filter'].rolling(10).max().iloc[6000:7000].plot(ax=ax, color=tableau20[4])
#



In [365]:
#data[data['mf_dif'].abs() > 15]
data['mf_dif']


Out[365]:
ts
2016-08-12 10:46:25+02:00      NaN
2016-08-12 10:46:26+02:00      NaN
2016-08-12 10:46:27+02:00      NaN
2016-08-12 10:46:28+02:00      NaN
2016-08-12 10:46:29+02:00      NaN
2016-08-12 10:46:30+02:00      NaN
2016-08-12 10:46:31+02:00      NaN
2016-08-12 10:46:32+02:00      NaN
2016-08-12 10:46:33+02:00      NaN
2016-08-12 10:46:34+02:00      0.0
2016-08-12 10:46:35+02:00      0.0
2016-08-12 10:46:36+02:00      0.0
2016-08-12 10:46:37+02:00      0.0
2016-08-12 10:46:38+02:00      0.0
2016-08-12 10:46:39+02:00      0.0
2016-08-12 10:46:40+02:00      0.0
2016-08-12 10:46:41+02:00      0.0
2016-08-12 10:46:42+02:00      0.0
2016-08-12 10:46:43+02:00      0.0
2016-08-12 10:46:44+02:00      0.0
2016-08-12 10:46:45+02:00      0.0
2016-08-12 10:46:46+02:00      0.0
2016-08-12 10:46:47+02:00      0.0
2016-08-12 10:46:48+02:00      0.0
2016-08-12 10:46:49+02:00      0.0
2016-08-12 10:46:50+02:00      0.0
2016-08-12 10:46:51+02:00      0.0
2016-08-12 10:46:52+02:00      0.0
2016-08-12 10:46:53+02:00      0.0
2016-08-12 10:46:54+02:00   -656.0
                             ...  
2016-09-14 13:19:23+02:00      1.0
2016-09-14 13:19:24+02:00      1.0
2016-09-14 13:19:25+02:00      0.0
2016-09-14 13:19:26+02:00      0.0
2016-09-14 13:19:27+02:00      0.0
2016-09-14 13:19:28+02:00      0.0
2016-09-14 13:19:29+02:00      0.0
2016-09-14 13:19:30+02:00      0.0
2016-09-14 13:19:31+02:00      0.0
2016-09-14 13:19:32+02:00      0.0
2016-09-14 13:19:33+02:00      1.0
2016-09-14 13:19:34+02:00      1.0
2016-09-14 13:19:35+02:00      1.0
2016-09-14 13:19:36+02:00      1.0
2016-09-14 13:19:37+02:00      1.0
2016-09-14 13:19:38+02:00      1.0
2016-09-14 13:19:39+02:00      1.0
2016-09-14 13:19:40+02:00      1.0
2016-09-14 13:19:41+02:00      1.0
2016-09-14 13:19:42+02:00      0.0
2016-09-14 13:19:43+02:00      0.0
2016-09-14 13:19:44+02:00      0.0
2016-09-14 13:19:45+02:00      0.0
2016-09-14 13:19:46+02:00      0.0
2016-09-14 13:19:47+02:00      0.0
2016-09-14 13:19:48+02:00      0.0
2016-09-14 13:19:49+02:00      0.0
2016-09-14 13:19:50+02:00      0.0
2016-09-14 13:19:51+02:00      0.0
2016-09-14 13:19:52+02:00      0.0
Freq: S, Name: mf_dif, dtype: float64

In [304]:
saltos = pois[['seconds', 'alt_sun_fin', 'fin', 'pendiente_f', 'pendiente_last_mf', 
               'ts_fin', 'step', 'pendiente_salto', 'is_poi_to_label']].copy()
saltos


Out[304]:
seconds alt_sun_fin fin pendiente_f pendiente_last_mf ts_fin step pendiente_salto is_poi_to_label
intervalo
1.0 8 40.0 656.0 0.000000 0.000000 2016-08-12 10:47:01 0.0 0.000000 False
2.0 3356 50.0 525.0 -1.737374 -0.295492 2016-08-12 11:42:57 87.0 -1.737374 True
3.0 8 50.0 438.0 0.000000 0.000000 2016-08-12 11:43:05 0.0 -2.351351 False
4.0 37 50.0 472.0 2.351351 2.388889 2016-08-12 11:43:42 -56.0 1.351351 False
5.0 4 50.0 528.0 1.000000 0.666667 2016-08-12 11:43:46 -58.0 -8.571429 False
6.0 7 50.0 575.0 9.571429 10.166667 2016-08-12 11:43:53 -54.0 9.421429 False
7.0 1044 53.0 569.0 -1.818182 -0.293823 2016-08-12 12:01:17 152.0 -0.363636 True
8.0 11 53.0 417.0 -1.454545 -1.600000 2016-08-12 12:01:28 4.0 -1.414545 False
9.0 466 54.0 407.0 -1.141414 -0.187097 2016-08-12 12:09:14 131.0 -1.030303 True
10.0 9 54.0 276.0 -0.111111 -0.125000 2016-08-12 12:09:23 0.0 -0.311111 False
11.0 273 55.0 444.0 1.747475 0.720588 2016-08-12 12:13:56 -38.0 1.767475 False
12.0 306 56.0 448.0 -0.424242 -0.167213 2016-08-12 12:19:02 58.0 -0.142992 True
13.0 96 56.0 381.0 -0.281250 -0.273684 2016-08-12 12:20:38 15.0 4.385417 False
14.0 3 56.0 373.0 -4.666667 -1.000000 2016-08-12 12:20:41 27.0 -3.380952 False
15.0 7 56.0 348.0 -1.285714 -1.333333 2016-08-12 12:20:48 32.0 8.047619 False
16.0 3 56.0 330.0 -9.333333 -7.500000 2016-08-12 12:20:51 31.0 -9.393333 False
17.0 353 57.0 419.0 -0.949495 0.190341 2016-08-12 12:26:44 61.0 0.550505 True
18.0 6 57.0 359.0 -1.500000 -1.600000 2016-08-12 12:26:50 21.0 -2.106061 False
19.0 99 57.0 364.0 0.606061 0.530612 2016-08-12 12:28:29 -62.0 -1.018939 False
20.0 8 57.0 423.0 1.625000 1.142857 2016-08-12 12:28:37 -25.0 1.375000 False
21.0 516 59.0 358.0 -1.404040 -0.238835 2016-08-12 12:37:13 60.0 -2.494040 True
22.0 100 59.0 361.0 1.090000 1.080808 2016-08-12 12:38:53 -51.0 0.970000 False
23.0 701 60.0 543.0 -0.474747 0.146912 2016-08-12 12:50:34 71.0 2.525253 True
24.0 8 60.0 473.0 -3.000000 -3.285714 2016-08-12 12:50:42 38.0 -4.125000 False
25.0 56 60.0 475.0 1.125000 1.000000 2016-08-12 12:51:38 -59.0 0.500000 False
26.0 8 60.0 534.0 0.625000 0.571429 2016-08-12 12:51:46 -6.0 2.047535 False
27.0 71 61.0 498.0 -1.422535 -1.271429 2016-08-12 12:52:57 88.0 0.077465 False
28.0 10 61.0 411.0 -1.500000 -1.666667 2016-08-12 12:53:07 7.0 -1.470000 False
29.0 154 61.0 425.0 0.484848 0.261438 2016-08-12 12:55:41 -46.0 0.484848 False
30.0 8 61.0 471.0 0.000000 0.000000 2016-08-12 12:55:49 0.0 0.580000 False
... ... ... ... ... ... ... ... ... ...
903.0 4204 0.0 36.0 5.515152 0.911519 2016-09-13 23:14:42 -548.0 5.390152 True
904.0 8 0.0 583.0 0.125000 0.142857 2016-09-13 23:14:50 -2.0 0.125000 False
905.0 579 0.0 373.0 -5.484848 -0.944637 2016-09-13 23:24:29 337.0 -5.474848 True
906.0 37376 22.0 324.0 0.878788 0.171953 2016-09-14 09:47:25 -23.0 -0.454545 True
907.0 3 22.0 345.0 1.333333 1.000000 2016-09-14 09:47:28 -20.0 1.213333 False
908.0 4486 36.0 543.0 -0.171717 -0.036728 2016-09-14 11:02:14 -20.0 -0.231717 False
909.0 2174 42.0 535.0 -0.515152 -0.098497 2016-09-14 11:38:28 43.0 -0.515152 True
910.0 8 42.0 492.0 0.000000 0.000000 2016-09-14 11:38:36 0.0 -0.640000 False
911.0 1251 45.0 487.0 -0.717172 -0.125209 2016-09-14 11:59:27 61.0 -1.508838 True
912.0 24 45.0 441.0 0.791667 0.130435 2016-09-14 11:59:51 -28.0 0.791667 False
913.0 8 45.0 469.0 0.000000 0.000000 2016-09-14 11:59:59 0.0 -0.680000 False
914.0 1078 47.0 495.0 -0.747475 -0.130217 2016-09-14 12:17:57 119.0 -0.427475 True
915.0 315 48.0 375.0 0.000000 -0.003185 2016-09-14 12:23:12 -83.0 0.000000 True
916.0 8 48.0 458.0 0.000000 0.000000 2016-09-14 12:23:20 0.0 1.000000 False
917.0 32 48.0 458.0 -1.000000 -0.451613 2016-09-14 12:23:52 37.0 -1.000000 False
918.0 8 48.0 421.0 0.000000 0.000000 2016-09-14 12:24:00 0.0 -1.710526 False
919.0 38 48.0 470.0 1.710526 1.729730 2016-09-14 12:24:38 -19.0 1.520526 False
920.0 569 49.0 510.0 -0.101010 0.033451 2016-09-14 12:34:07 43.0 7.298990 True
921.0 5 49.0 477.0 -7.400000 -8.750000 2016-09-14 12:34:12 19.0 -5.600000 False
922.0 5 49.0 459.0 -1.800000 -1.000000 2016-09-14 12:34:17 20.0 -1.580000 False
923.0 513 50.0 394.0 0.404040 -0.050781 2016-09-14 12:42:50 -53.0 -0.795960 True
924.0 5 50.0 446.0 1.200000 0.250000 2016-09-14 12:42:55 -27.0 1.628571 False
925.0 70 50.0 472.0 -0.428571 -0.144928 2016-09-14 12:44:05 52.0 -0.053571 False
926.0 8 50.0 420.0 -0.375000 -0.428571 2016-09-14 12:44:13 3.0 -1.045000 False
927.0 190 50.0 477.0 -0.191919 0.238095 2016-09-14 12:47:23 37.0 -0.121919 False
928.0 138 50.0 464.0 0.505051 0.189781 2016-09-14 12:49:41 -26.0 0.505051 False
929.0 8 50.0 490.0 0.000000 0.000000 2016-09-14 12:49:49 0.0 0.060000 False
930.0 172 51.0 479.0 -0.313131 -0.210526 2016-09-14 12:52:41 32.0 -0.773131 False
931.0 145 51.0 489.0 0.101010 0.125000 2016-09-14 12:55:06 38.0 0.101010 False
932.0 8 51.0 451.0 0.000000 0.000000 2016-09-14 12:55:14 0.0 -0.140000 False

932 rows × 9 columns


In [172]:
ax = ldr_raw.iloc[::5].plot(figsize=FS, lw=1, color='darkorange', alpha=.9)
    (dplot['altitud'] * 12).plot(ax=ax, lw=4, color='y', alpha=.6)
    dplot['median_filter'].plot(ax=ax, color='darkred', lw=2, alpha=.7)


Out[172]:
dict_keys([datetime.date(2016, 9, 12), datetime.date(2016, 8, 23), datetime.date(2016, 8, 20), datetime.date(2016, 8, 12), datetime.date(2016, 8, 22), datetime.date(2016, 9, 3), datetime.date(2016, 8, 29), datetime.date(2016, 8, 15), datetime.date(2016, 9, 1), datetime.date(2016, 9, 10), datetime.date(2016, 9, 9), datetime.date(2016, 9, 6), datetime.date(2016, 9, 5), datetime.date(2016, 8, 31), datetime.date(2016, 9, 7), datetime.date(2016, 9, 14), datetime.date(2016, 8, 24), datetime.date(2016, 9, 8), datetime.date(2016, 9, 2), datetime.date(2016, 8, 14), datetime.date(2016, 8, 21), datetime.date(2016, 9, 11), datetime.date(2016, 8, 18), datetime.date(2016, 8, 25), datetime.date(2016, 8, 13), datetime.date(2016, 8, 19), datetime.date(2016, 9, 4), datetime.date(2016, 8, 17), datetime.date(2016, 8, 26), datetime.date(2016, 8, 16), datetime.date(2016, 9, 13), datetime.date(2016, 8, 27), datetime.date(2016, 8, 30), datetime.date(2016, 8, 28)])

In [178]:
from pykeyboard import PyKeyboard, mac

k = PyKeyboard()

k.press_key('command')
k.tap_key('tab')
k.release_key('command')

In [54]:
#data_homog.ldr.between_time().rolling(75, center=True, win_type='triang').sum().plot()
t0, tf = '0:00', '1:00'
ks_1, ks_2 = 301, 75
df = data_homog.ldr.between_time(t0, tf)
#ax = df.plot(figsize=FS, color=tableau20[6], lw=.75, alpha=.7)
s1 = pd.Series(medfilt(df, kernel_size=ks_1), index=df.index, 
              name='Medfilt_301').resample('5s').median() #.plot(ax=ax, color=tableau20[4], lw=4, alpha=.5)
s2 = pd.Series(medfilt(df, kernel_size=ks_2), index=df.index, 
              name='Medfilt_301').resample('5s').median() #.plot(ax=ax, color=tableau20[2], lw=3, alpha=.6)
s3 = pd.Series(medfilt(df, kernel_size=11), index=df.index, 
              name='Medfilt_301').resample('5s').median() #.plot(ax=ax, color=tableau20[8], lw=2, alpha=.6)

ax = (s2 - s1).plot(figsize=FS, color=tableau20[6], lw=.75, alpha=.7)
(s3 - s2).plot(ax=ax, color=tableau20[4], lw=1.5, alpha=.7)


Out[54]:
<matplotlib.axes._subplots.AxesSubplot at 0x12167c748>

Filtrado de LDR e identificación de POI's

  • Resample a 1s para homogeneizar time series
  • Determinación de día / noche con offset de 5 min
  • Median filter (scipy.signal.medfilt) con W=5min

In [3]:
def _es_max_local(x):
    mid = len(x) // 2
    if (np.argmax(x) == mid) and (x[mid] > 10):
        return True
    return False

def _pendiente(x):
    return (x[-1] - x[0]) / len(x)


def _pendiente_ini(x):
    if len(x) > 100:
        return _pendiente(x.values[1:101])
    return _pendiente(x.values)

def _pendiente_fin(x):
    if len(x) > 100:
        return _pendiente(x.values[-100:-1])
    return _pendiente(x.values)

def _pendiente_last_gen(x):
    limit = min(len(x), 600)
    return _pendiente(x.values[-limit:-1])

def _get_alt(d):
    alt = pysolar.solar.get_altitude_fast(LAT, LONG, d)
    if alt > 0:
        return alt
    return 0


# Resampling 1s + sun
data_homog = data_raw.resample('1s').mean().interpolate().dropna()
data_homog = set_sun_times(data_homog, tz=TZ, lat_deg=LAT, long_deg=LONG, offset='5min')
subset_pysolar = data_homog.iloc[::120].index
data_homog = data_homog.join(pd.Series(subset_pysolar, index=subset_pysolar, name='altitud'
                                      ).apply(_get_alt)).interpolate()

# Median filter
data_homog['median_filter'] = scipy.signal.medfilt(data_homog.ldr, 301)

# BUSCA POI's
data_homog['r_disp'] = data_homog.median_filter.rolling(5, center=True).apply(lambda x: x.max() - x.min()).fillna(0)
data_homog['poi'] = data_homog['r_disp'].rolling(5).apply(_es_max_local).fillna(False).astype(bool)

# Construye intervalos entre POI's
data_homog['intervalo'] = data_homog['poi'].cumsum()

# DATOS POIS:
gb_poi = data_homog.tz_localize(None).reset_index().groupby('intervalo')
pois = pd.concat([gb_poi.sun.apply(lambda x: np.sum(x) / len(x)).rename('fr_sun'),
                  gb_poi.ldr.count().rename('seconds'),
                  gb_poi.altitud.mean(),
                  gb_poi.median_filter.apply(lambda x: (x.values[-1] - x.values[0]) / len(x)).rename('pendiente_tot'),
                  gb_poi.median_filter.apply(lambda x: np.round(np.median(x.values[1:10]))).rename('inicio'),
                  gb_poi.median_filter.apply(lambda x: np.round(np.median(x.values[-10:-1]))).rename('fin'),
                  gb_poi.median_filter.apply(_pendiente_ini).rename('pendiente_i'),
                  gb_poi.median_filter.apply(_pendiente_fin).rename('pendiente_f'),
                  gb_poi.median_filter.apply(_pendiente_last_gen).rename('pendiente_last_mf'),
                  gb_poi.ts.first().rename('ts_ini'), 
                  gb_poi.ts.last().rename('ts_fin')], axis=1)
pois['step'] = (pois['inicio'] - pois['fin'].shift()) # .fillna(0)
pois.head(10)


Out[3]:
fr_sun seconds altitud pendiente_tot inicio fin pendiente_i pendiente_f pendiente_last_mf ts_ini ts_fin step
intervalo
0 0.203878 34344 1.823551 0.012520 31.0 438.0 0.190000 0.202020 0.056761 2016-09-12 00:00:00 2016-09-12 09:32:23 NaN
1 1.000000 953 21.597301 -0.060860 456.0 510.0 0.140000 -0.303030 -0.016694 2016-09-12 09:32:24 2016-09-12 09:48:16 18.0
2 1.000000 832 24.397394 0.115385 334.0 416.0 -0.040000 0.474747 0.160267 2016-09-12 09:48:17 2016-09-12 10:02:08 -176.0
3 1.000000 17 25.717194 9.000000 447.0 449.0 9.000000 9.000000 7.500000 2016-09-12 10:02:09 2016-09-12 10:02:25 31.0
4 1.000000 4912 33.056631 0.006311 602.0 697.0 0.210000 -0.565657 -0.113523 2016-09-12 10:02:26 2016-09-12 11:24:17 153.0
5 1.000000 5979 46.829312 -0.019401 609.0 505.0 -0.060000 -0.020202 0.020033 2016-09-12 11:24:18 2016-09-12 13:03:56 -88.0
6 1.000000 13935 49.167678 -0.007320 493.0 391.0 -0.010000 0.010101 -0.030050 2016-09-12 13:03:57 2016-09-12 16:56:11 -12.0
7 1.000000 3 36.159712 0.333333 408.0 408.0 0.333333 0.333333 0.000000 2016-09-12 16:56:12 2016-09-12 16:56:14 17.0
8 1.000000 56 36.076439 0.500000 418.0 430.0 0.500000 0.500000 0.490909 2016-09-12 16:56:15 2016-09-12 16:57:10 10.0
9 1.000000 6086 26.899149 -0.010845 443.0 365.0 0.010000 0.101010 -0.005008 2016-09-12 16:57:11 2016-09-12 18:38:36 13.0

In [4]:
# PLOT
def plot_data_ldr(data_homog):
    dplot = data_homog.tz_localize(None)

    ax = dplot.ldr.iloc[::5].plot(figsize=FS, lw=1, color='darkorange', alpha=.9)
    (dplot['altitud'].iloc[::10] * 12).plot(ax=ax, lw=4, color='y', alpha=.6)
    dplot['median_filter'].iloc[::5].plot(ax=ax, color='darkred', lw=2, alpha=.7)
    (dplot['intervalo'] * 20).plot(ax=ax, lw=3, color='magenta', alpha=.6)

    ylim = ax.get_ylim()[1]
    ax.vlines(dplot[dplot['poi']].index, 0, ylim, lw=2, color='darkgrey', alpha=.8, linestyle='--')
    
    ax.xaxis.set_major_locator(mpd.HourLocator())
    ax.xaxis.set_major_formatter(mpd.DateFormatter('%H:%M'))
    return ax
    

ax = plot_data_ldr(data_homog)

ylim = ax.get_ylim()[1]
intervalos_largos = pois[pois.seconds > 600].copy()
intervalos_largos['ts'] = intervalos_largos['ts_ini'] + (intervalos_largos['ts_fin'] - intervalos_largos['ts_ini']) / 2
for i, t in intervalos_largos['ts'].items():
    ax.annotate(str(i), (t, ylim - 50), color='red', ha='center')
plt.show()



In [ ]:


In [7]:
#momentos_sol = gb_poi.sun.apply(lambda x: np.sum(x)).rename('fr_sun').cumsum()
#(momentos_sol.max() - momentos_sol) / momentos_sol.max()

intervalo_luz_natural = []
intervalo_luz_artificial_principal = []

offset_step = pd.Timedelta('5min')
d_calc = data_homog.tz_localize(None)
ymax = ((d_calc.median_filter.max() + 100) // 100) * 100
for interv, row in pois.iterrows():
    print_secc('Intervalo {}. De {:%d/%m/%y %H:%M:%S} a {:%H:%M:%S} ({} segundos)'
               .format(interv, row.ts_ini, row.ts_fin, row.seconds))
    delta_interv = row.ts_fin - row.ts_ini
    offset = delta_interv / 10
    df_slice = d_calc.loc[row.ts_ini - offset: row.ts_fin + offset]
    ax = plot_data_ldr(df_slice)
    #ax =plt.gca()
    ini_p = mpd.date2num(row.ts_ini - offset)
    fin_p = mpd.date2num(row.ts_fin + offset)
    print(fin_p - ini_p)
    #ax.add_patch(mpp.Rectangle((ini_p, 0), fin_p - ini_p, ymax, angle=0.0, color='red', linewidth=100, fill=True))
    #ax.fill_betweenx([0, ymax], mpd.date2num(row.ts_ini - offset), mpd.date2num(row.ts_fin + offset), alpha=.6, color=tableau20[1])
    #ax.fill_betweenx([0, ymax], row.ts_ini - offset, row.ts_fin + offset, alpha=.6, color=tableau20[1])
    
    ax.fill_betweenx([0, ymax], row.ts_fin - offset_step, row.ts_fin + offset_step, alpha=.4, color=tableau20[1])
    #ax.autoscale()
    break

#dplot


 ==> Intervalo 0. De 12/09/16 00:00:00 a 09:32:23 (34344 segundos)
0.4769861111417413

In [9]:
def _str_to_bool(resp):
    resp = resp.lower().rstrip().lstrip()
    if resp in ['si', 'sí', 's', '1', 'yes', 'y']:
        return True
    return False
    

def _get_input():
    es_nat = _str_to_bool(input('Es intervalo natural? '))
    es_art = _str_to_bool(input('Es cambio por luz artificial? '))
    return es_nat, es_art

_get_input()


Es intervalo natural? s
Es cambio por luz artificial? n
Out[9]:
(True, False)

In [290]:
pois[pois.pendiente_last_mf.abs() < 1]


Out[290]:
intervalo fr_sun seconds alt_sun_ini alt_sun_fin altitud ldr_mean ldr_median pendiente_tot inicio fin pendiente_i pendiente_f pendiente_last_mf ts_ini step is_poi_to_label solo_natural artificial_princ
ts_fin
2016-08-12 10:46:53 0.0 1.000000 145.0 40.0 40.0 40.0 0.000000 0.0 0.000000 0.0 0.0 0.000000 0.000000 0.000000 2016-08-12 10:46:25 -656.0 False NaN NaN
2016-08-12 11:42:48 1.0 1.000000 16775.0 40.0 50.0 45.0 646.824143 648.0 -0.028912 656.0 610.0 0.000000 -0.616162 -0.110184 2016-08-12 10:46:54 102.0 True True False
2016-08-12 11:43:38 3.0 1.000000 205.0 50.0 50.0 50.0 440.536585 438.0 1.365854 438.0 438.0 1.365854 1.365854 0.850000 2016-08-12 11:42:58 -90.0 False NaN NaN
2016-08-12 11:59:49 5.0 1.000000 4795.0 50.0 53.0 51.0 630.085506 630.0 0.030240 622.0 631.0 0.320000 -0.030303 -0.005008 2016-08-12 11:43:51 17.0 False NaN NaN
2016-08-12 12:01:14 6.0 1.000000 425.0 53.0 53.0 53.0 610.741176 614.0 -1.435294 614.0 612.0 -1.435294 -1.435294 -0.976190 2016-08-12 11:59:50 192.0 True True False
2016-08-12 12:09:13 7.0 1.000000 2395.0 53.0 54.0 54.0 422.672234 426.0 -0.286013 420.0 408.0 -0.410000 -0.757576 -0.207113 2016-08-12 12:01:15 132.0 True True False
2016-08-12 12:13:06 8.0 1.000000 1165.0 54.0 55.0 55.0 296.549356 297.0 0.437768 276.0 372.0 0.140000 0.898990 0.431034 2016-08-12 12:09:14 -23.0 False NaN NaN
2016-08-12 12:19:02 10.0 1.000000 1550.0 55.0 56.0 55.0 470.993548 471.0 -0.174194 478.0 448.0 0.050000 -0.424242 -0.139159 2016-08-12 12:13:53 58.0 True True False
2016-08-12 12:20:42 11.0 1.000000 500.0 56.0 56.0 56.0 384.020000 385.0 -0.500000 390.0 375.0 -0.500000 -0.500000 -0.414141 2016-08-12 12:19:03 33.0 True True False
2016-08-12 12:26:43 13.0 1.000000 1760.0 56.0 57.0 57.0 386.988636 409.0 0.190341 299.0 420.0 0.060000 -0.818182 0.227920 2016-08-12 12:20:52 66.0 True True False
2016-08-12 12:37:09 14.0 1.000000 3130.0 57.0 59.0 58.0 432.172524 455.0 -0.054313 354.0 415.0 0.020000 -0.929293 0.075125 2016-08-12 12:26:44 114.0 True True False
2016-08-12 12:38:50 15.0 1.000000 505.0 59.0 59.0 59.0 294.396040 289.0 0.821782 301.0 311.0 0.870000 0.757576 0.710000 2016-08-12 12:37:10 -101.0 True True False
2016-08-12 12:50:34 16.0 1.000000 3520.0 59.0 60.0 60.0 459.451705 420.0 0.133523 412.0 543.0 0.160000 -0.474747 0.146912 2016-08-12 12:38:51 73.0 True True False
2016-08-12 12:51:28 18.0 1.000000 215.0 60.0 60.0 60.0 435.860465 435.0 0.255814 435.0 435.0 0.255814 0.255814 0.119048 2016-08-12 12:50:46 -49.0 False NaN NaN
2016-08-12 12:52:53 19.0 1.000000 425.0 60.0 61.0 61.0 536.705882 548.0 0.235294 484.0 547.0 0.235294 0.235294 0.500000 2016-08-12 12:51:29 104.0 True True False
2016-08-12 12:55:41 21.0 1.000000 815.0 61.0 61.0 61.0 405.877301 401.0 0.263804 410.0 425.0 -0.100000 0.484848 0.203704 2016-08-12 12:52:59 -46.0 True True False
2016-08-12 12:56:22 22.0 1.000000 205.0 61.0 61.0 61.0 470.390244 471.0 -0.341463 471.0 471.0 -0.341463 -0.341463 -0.200000 2016-08-12 12:55:42 22.0 False NaN NaN
2016-08-12 12:56:43 23.0 1.000000 105.0 61.0 61.0 61.0 446.428571 448.0 -0.952381 449.0 444.0 -0.952381 -0.952381 -0.550000 2016-08-12 12:56:23 26.0 False NaN NaN
2016-08-12 13:04:06 24.0 1.000000 2215.0 61.0 62.0 62.0 432.920993 428.0 0.227991 418.0 470.0 -0.070000 0.585859 0.203620 2016-08-12 12:56:44 -70.0 True True False
2016-08-12 13:43:55 25.0 1.000000 11945.0 62.0 65.0 64.0 498.529092 495.0 -0.025534 540.0 495.0 -0.020000 -0.303030 -0.031720 2016-08-12 13:04:07 37.0 True True False
2016-08-12 13:49:22 27.0 1.000000 1580.0 65.0 65.0 65.0 378.047468 373.0 0.167722 404.0 418.0 -0.400000 0.858586 0.133333 2016-08-12 13:44:07 -60.0 True True False
2016-08-12 14:20:47 28.0 1.000000 9425.0 65.0 66.0 66.0 495.267374 515.0 0.008488 478.0 466.0 0.360000 0.404040 0.068447 2016-08-12 13:49:23 -60.0 True True False
2016-08-12 14:23:43 29.0 1.000000 880.0 66.0 66.0 66.0 569.562500 585.0 -0.045455 526.0 523.0 0.740000 -0.878788 -0.045714 2016-08-12 14:20:48 30.0 False NaN NaN
2016-08-12 14:26:35 32.0 1.000000 590.0 66.0 65.0 65.0 361.542373 360.0 -0.228814 389.0 367.0 -0.560000 0.101010 -0.299145 2016-08-12 14:24:38 -32.0 True True False
2016-08-12 14:28:59 33.0 1.000000 720.0 65.0 65.0 65.0 469.770833 484.0 0.493056 399.0 486.0 1.080000 0.171717 0.503497 2016-08-12 14:26:36 26.0 False NaN NaN
2016-08-12 14:36:06 34.0 1.000000 2135.0 65.0 65.0 65.0 525.976581 542.0 0.117096 460.0 500.0 0.820000 -0.404040 0.103286 2016-08-12 14:29:00 -27.0 False NaN NaN
2016-08-12 14:38:45 36.0 1.000000 740.0 65.0 65.0 65.0 547.236486 552.0 -0.500000 552.0 523.0 -0.010000 -0.616162 -0.408163 2016-08-12 14:36:18 58.0 True True False
2016-08-12 14:41:37 38.0 1.000000 800.0 65.0 65.0 65.0 361.337500 359.0 0.475000 391.0 398.0 -0.420000 1.434343 0.459119 2016-08-12 14:38:58 -100.0 True True False
2016-08-12 14:46:27 39.0 1.000000 1450.0 65.0 64.0 65.0 551.368966 559.0 -0.086207 498.0 553.0 0.800000 -0.444444 0.072664 2016-08-12 14:41:38 151.0 True True False
2016-08-12 14:47:24 40.0 1.000000 285.0 64.0 64.0 64.0 398.105263 393.0 0.508772 402.0 402.0 0.508772 0.508772 0.250000 2016-08-12 14:46:28 -60.0 False NaN NaN
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
2016-08-12 16:13:06 80.0 1.000000 1085.0 54.0 53.0 54.0 257.373272 244.0 0.101382 313.0 274.0 -0.990000 0.858586 -0.083333 2016-08-12 16:09:30 -136.0 True True False
2016-08-12 16:32:16 82.0 1.000000 5710.0 53.0 50.0 52.0 538.449212 538.0 0.017513 554.0 544.0 -0.040000 0.363636 0.040067 2016-08-12 16:13:15 -31.0 True True False
2016-08-12 18:18:47 83.0 1.000000 31955.0 50.0 30.0 40.0 486.309341 485.0 -0.026913 575.0 383.0 0.100000 0.101010 -0.008347 2016-08-12 16:32:17 -22.0 True True False
2016-08-12 18:58:37 84.0 1.000000 11950.0 30.0 22.0 26.0 383.106276 382.0 -0.039749 405.0 339.0 0.000000 -0.595960 -0.118531 2016-08-12 18:18:48 41.0 True False False
2016-08-12 19:04:02 86.0 1.000000 1510.0 22.0 21.0 22.0 207.288079 208.0 0.052980 211.0 206.0 -0.030000 0.202020 0.046512 2016-08-12 18:59:01 -30.0 False NaN NaN
2016-08-12 20:59:16 87.0 1.000000 34570.0 21.0 0.0 10.0 156.585189 165.0 -0.013740 236.0 59.0 0.100000 0.555556 0.070117 2016-08-12 19:04:03 -97.0 True True False
2016-08-12 21:02:09 88.0 1.000000 865.0 0.0 0.0 0.0 174.271676 176.0 -0.398844 156.0 144.0 0.340000 -0.818182 -0.226744 2016-08-12 20:59:17 90.0 True False False
2016-08-12 21:04:43 89.0 1.000000 770.0 0.0 0.0 0.0 54.753247 53.0 0.714286 54.0 54.0 -0.020000 0.787879 0.483660 2016-08-12 21:02:10 -134.0 True True False
2016-08-12 21:12:52 90.0 1.000000 2445.0 0.0 0.0 0.0 189.460123 189.0 -0.173824 188.0 179.0 0.150000 -0.414141 -0.069672 2016-08-12 21:04:44 102.0 True False False
2016-08-12 21:14:38 92.0 1.000000 195.0 0.0 0.0 0.0 179.897436 181.0 0.435897 181.0 181.0 0.435897 0.435897 0.447368 2016-08-12 21:14:00 17.0 False NaN NaN
2016-08-12 21:16:24 94.0 1.000000 485.0 0.0 0.0 0.0 67.865979 69.0 -0.484536 74.0 57.0 -0.484536 -0.484536 -0.479167 2016-08-12 21:14:48 22.0 False NaN NaN
2016-08-13 00:09:59 95.0 0.014882 52075.0 0.0 0.0 0.0 32.328373 32.0 0.002016 35.0 33.0 -0.030000 0.141414 0.025042 2016-08-12 21:16:25 -34.0 True True True
2016-08-13 00:29:31 97.0 0.000000 5810.0 0.0 0.0 0.0 592.024957 592.0 -0.453528 593.0 588.0 0.150000 -3.282828 -0.545910 2016-08-13 00:10:10 554.0 True False True
2016-08-13 09:17:19 98.0 0.263199 158340.0 0.0 22.0 2.0 56.757515 32.0 0.007863 34.0 274.0 -0.050000 0.191919 0.058431 2016-08-13 00:29:32 -25.0 True True False
2016-08-13 19:49:45 99.0 1.000000 189730.0 22.0 12.0 45.0 488.277895 529.0 0.004717 299.0 180.0 -0.020000 2.929293 0.467446 2016-08-13 09:17:20 -345.0 True True True
2016-08-13 20:24:56 101.0 1.000000 10435.0 12.0 6.0 9.0 567.860086 567.0 -0.206037 577.0 562.0 0.020000 -2.080808 -0.347245 2016-08-13 19:50:10 446.0 True False True
2016-08-13 21:00:22 102.0 1.000000 10630.0 6.0 0.0 2.0 82.995767 83.0 -0.033396 116.0 36.0 -0.030000 -0.202020 -0.051753 2016-08-13 20:24:57 -17.0 True True False
2016-08-13 21:00:25 103.0 1.000000 15.0 0.0 0.0 0.0 53.000000 53.0 0.000000 53.0 53.0 0.000000 0.000000 0.000000 2016-08-13 21:00:23 54.0 False NaN NaN
2016-08-13 22:16:33 104.0 0.227452 22840.0 0.0 0.0 0.0 -1.000000 -1.0 0.000000 -1.0 -1.0 0.000000 0.000000 0.000000 2016-08-13 21:00:26 -560.0 True True True
2016-08-13 22:19:39 105.0 0.000000 930.0 0.0 0.0 0.0 562.564516 563.0 0.005376 559.0 562.0 0.040000 -0.020202 0.010811 2016-08-13 22:16:34 563.0 True False True
2016-08-13 22:20:39 106.0 0.000000 300.0 0.0 0.0 0.0 8.383333 -1.0 9.383333 -1.0 -1.0 9.383333 9.383333 0.000000 2016-08-13 22:19:40 -564.0 True True True
2016-08-13 22:37:30 107.0 0.000000 5055.0 0.0 0.0 0.0 565.566766 566.0 -0.398615 563.0 563.0 0.000000 -1.626263 -0.270451 2016-08-13 22:20:40 521.0 True False True
2016-08-13 23:39:13 108.0 0.000000 18515.0 0.0 0.0 0.0 40.711585 41.0 0.137726 42.0 38.0 -0.060000 4.151515 0.682805 2016-08-13 22:37:31 -535.0 True True True
2016-08-13 23:58:25 109.0 0.000000 5760.0 0.0 0.0 0.0 568.312500 569.0 -0.426215 573.0 564.0 0.020000 -2.323232 -0.392321 2016-08-13 23:39:14 527.0 True False True
2016-08-14 12:02:26 110.0 0.418660 217205.0 0.0 53.0 10.0 176.279091 35.0 0.014848 37.0 605.0 -0.070000 0.393939 0.051753 2016-08-13 23:58:26 -89.0 True True False
2016-08-14 14:28:35 111.0 1.000000 43845.0 53.0 65.0 61.0 657.709431 656.0 -0.013342 694.0 670.0 0.080000 -0.303030 -0.021703 2016-08-14 12:02:27 125.0 True True False
2016-08-14 21:53:39 112.0 0.916492 133520.0 65.0 0.0 31.0 343.893200 329.0 0.000037 545.0 76.0 0.020000 3.141414 0.577629 2016-08-14 14:28:36 -499.0 True True True
2016-08-14 22:21:31 113.0 0.000000 8360.0 0.0 0.0 0.0 569.935407 569.0 0.003589 575.0 567.0 0.000000 0.131313 0.020033 2016-08-14 21:53:40 -18.0 True False False
2016-08-14 22:25:24 114.0 0.000000 1165.0 0.0 0.0 0.0 584.309013 584.0 -0.030043 585.0 581.0 0.030000 -0.060606 -0.012931 2016-08-14 22:21:32 18.0 False NaN NaN
2016-08-14 22:59:56 116.0 0.000000 10285.0 0.0 0.0 0.0 45.341274 45.0 0.210015 46.0 47.0 -0.180000 2.595960 0.427379 2016-08-14 22:25:40 -523.0 True True True

78 rows × 19 columns


In [196]:
def _rs_raw(x):
    n = len(x)
    return np.round(np.sum(x) / n**2, 0) * n
    #data_homog.rolling(5, center=True, min_periods=1).apply(_rs_raw).plot(figsize=FS)

In [197]:
# pysolar.util.diffuse_underclear(latitude_deg, longitude_deg, when, elevation=0.0, temperature=288.15,

In [7]:
def _calc_pendiente(serie_homogenea, roll_window=3):
    return serie_homogenea.diff().rolling(roll_window, center=True
                                         ).apply(lambda x: (x[-1] - x[0]) / roll_window
                                                ).rename('{}_roll_{}'.format(serie_homogenea.name, roll_window))


pd.concat([_calc_pendiente(data_homog.ldr, 5), 
           _calc_pendiente(data_homog.ldr, 30), 
           _calc_pendiente(data_homog.ldr, 120)], axis=1).plot(figsize=FS, alpha=.5, lw=1)


Out[7]:
<matplotlib.axes._subplots.AxesSubplot at 0x1055e05c0>

In [8]:
ts_day = pd.Timestamp(data_homog.index[0].date(), tz=TZ)
print_yellowb(ts_day)

#pysolar.util.direct_underclear?
# NO -> pysolar.radiation.get_radiation_direct?

        
        


data_calc = pd.concat([data_homog, 
                       _calc_pendiente(data_homog.ldr, 5), 
                       _calc_pendiente(data_homog.ldr, 120), 
                       _calc_pendiente(data_homog.ldr, 6000)], axis=1)

#.plot(figsize=FS, alpha=.5, lw=1)


#data_calc['dif_30'] = data_calc['ldr_roll_5'] - data_calc['ldr_roll_30']
data_calc['dif_120'] = data_calc['ldr_roll_5'] - data_calc['ldr_roll_120']
data_calc['dif_6000'] = data_calc['ldr_roll_5'] - data_calc['ldr_roll_6000']
#ax = data_homog[~data_homog.sun].plot(figsize=FS, lw=1, color='violet')
#data_homog[data_homog.sun].plot(ax=ax, lw=2, color='darkorange')
#data_homog[data_homog.sun].plot(figsize=FS, lw=2, color='darkorange')

data_calc[data_calc.sun]['ldr_roll_6000'].plot(figsize=FS)
print(data_calc.head())
data_calc.iloc[:-25000:1000].tail(10)


2016-09-12 00:00:00+02:00
set_sun_times TOOK: 0.002 s
                            ldr    sun  ldr_roll_5  ldr_roll_120  ldr_roll_6000  dif_120  dif_6000
ts                                                                                                
2016-09-12 00:00:00+02:00  40.0  False         NaN           NaN            NaN      NaN       NaN
2016-09-12 00:00:01+02:00  40.0  False         NaN           NaN            NaN      NaN       NaN
2016-09-12 00:00:02+02:00  41.0  False         NaN           NaN            NaN      NaN       NaN
2016-09-12 00:00:03+02:00  41.0  False         0.2           NaN            NaN      NaN       NaN
2016-09-12 00:00:04+02:00  40.0  False        -0.4           NaN            NaN      NaN       NaN
Out[8]:
ldr sun ldr_roll_5 ldr_roll_120 ldr_roll_6000 dif_120 dif_6000
ts
2016-09-12 14:26:40+02:00 616.0 True -0.2 0.000000 0.000167 -0.200000 -0.200167
2016-09-12 14:43:20+02:00 626.0 True 0.0 0.000000 0.000000 0.000000 0.000000
2016-09-12 15:00:00+02:00 626.0 True 0.4 0.000000 0.000000 0.400000 0.400000
2016-09-12 15:16:40+02:00 609.0 True -0.4 0.000000 -0.000167 -0.400000 -0.399833
2016-09-12 15:33:20+02:00 590.0 True 0.0 0.000000 0.000167 0.000000 -0.000167
2016-09-12 15:50:00+02:00 556.0 True -0.2 0.000000 -0.000167 -0.200000 -0.199833
2016-09-12 16:06:40+02:00 521.0 True 0.0 0.000000 0.000167 0.000000 -0.000167
2016-09-12 16:23:20+02:00 472.0 True -0.2 -0.008333 -0.000333 -0.191667 -0.199667
2016-09-12 16:40:00+02:00 426.0 True -0.4 0.000000 0.000333 -0.400000 -0.400333
2016-09-12 16:56:40+02:00 430.0 True -0.2 0.016667 0.000000 -0.216667 -0.200000

In [ ]:


In [ ]:


In [9]:
rs = data_raw.ldr.resample('3s')
data_3 = pd.concat([rs.max().rename('ldr_max'), 
                    rs.min().rename('ldr_min'), 
                    rs.mean().rename('ldr_mean'), 
                    rs.median().rename('ldr_median')], axis=1)
data_3['pend'] = rs.apply(lambda x: (x[-1] - x[0]))
data_3['disp'] = data_3['ldr_max'] - data_3['ldr_min']

data_3 = set_sun_times(data_3)

data_3.head(20)


set_sun_times TOOK: 0.002 s
Out[9]:
ldr_max ldr_min ldr_mean ldr_median pend disp sun
ts
2016-09-12 00:00:00+02:00 41 40 40.333333 40.0 1 1 False
2016-09-12 00:00:03+02:00 41 40 40.666667 41.0 0 1 False
2016-09-12 00:00:06+02:00 42 40 41.000000 41.0 2 2 False
2016-09-12 00:00:09+02:00 42 41 41.666667 42.0 0 1 False
2016-09-12 00:00:12+02:00 42 40 41.000000 41.0 -1 2 False
2016-09-12 00:00:15+02:00 42 41 41.333333 41.0 -1 1 False
2016-09-12 00:00:18+02:00 41 40 40.666667 41.0 -1 1 False
2016-09-12 00:00:21+02:00 41 41 41.000000 41.0 0 0 False
2016-09-12 00:00:24+02:00 42 41 41.333333 41.0 -1 1 False
2016-09-12 00:00:27+02:00 41 41 41.000000 41.0 0 0 False
2016-09-12 00:00:30+02:00 41 41 41.000000 41.0 0 0 False
2016-09-12 00:00:33+02:00 42 41 41.333333 41.0 0 1 False
2016-09-12 00:00:36+02:00 41 41 41.000000 41.0 0 0 False
2016-09-12 00:00:39+02:00 41 40 40.666667 41.0 -1 1 False
2016-09-12 00:00:42+02:00 41 40 40.333333 40.0 1 1 False
2016-09-12 00:00:45+02:00 41 41 41.000000 41.0 0 0 False
2016-09-12 00:00:48+02:00 41 34 36.666667 35.0 -7 7 False
2016-09-12 00:00:51+02:00 31 21 27.333333 30.0 -1 10 False
2016-09-12 00:00:54+02:00 40 38 39.333333 40.0 2 2 False
2016-09-12 00:00:57+02:00 40 40 40.000000 40.0 0 0 False

In [198]:
#data_3['rms'] = rs.apply(lambda x: np.sqrt(np.mean(x**2)))
#data_3['rms_roll'] = data_3.rms.rolling(10, center=True).apply(lambda x: np.sqrt(np.mean(x**2)))
#data_3['delta_rms'] = data_3['rms'] - data_3['rms_roll']

In [ ]:


In [ ]:


In [36]:
delta = pd.Timedelta('10s')
for t in maximos.index:
    delta_ml = maximos.loc[t]
    y_ldr = data_homog.loc[t, 'ldr'] 
    y_mf = data_homog.loc[t, 'median_filter']
    with_sun = data_homog.loc[t, 'sun']
    
    mf_slice = data_homog.loc[t - delta:t + delta, ['ldr', 'median_filter']]
    
    print_info('{} -> {} (delta_ml={}, median_filter={}, sun={})\n{}'.format(t, y_ldr, delta_ml, y_mf, with_sun, mf_slice))
    #plt.plot(t, , marker='o', color='red', markersize=5)
    break
maximos


2016-09-12 09:32:22+02:00 -> 451.0 (delta_ml=12.0, median_filter=443.0, sun=True)
                             ldr  median_filter
ts                                             
2016-09-12 09:32:12+02:00  418.0          428.0
2016-09-12 09:32:13+02:00  407.0          428.0
2016-09-12 09:32:14+02:00  386.0          428.0
2016-09-12 09:32:15+02:00  401.0          428.0
2016-09-12 09:32:16+02:00  443.0          434.0
2016-09-12 09:32:17+02:00  464.0          438.0
2016-09-12 09:32:18+02:00  467.0          438.0
2016-09-12 09:32:19+02:00  467.0          438.0
2016-09-12 09:32:20+02:00  466.0          439.0
2016-09-12 09:32:21+02:00  463.0          442.0
2016-09-12 09:32:22+02:00  451.0          443.0
2016-09-12 09:32:23+02:00  438.0          451.0
2016-09-12 09:32:24+02:00  438.0          454.0
2016-09-12 09:32:25+02:00  439.0          455.0
2016-09-12 09:32:26+02:00  438.0          455.0
2016-09-12 09:32:27+02:00  434.0          455.0
2016-09-12 09:32:28+02:00  442.0          456.0
2016-09-12 09:32:29+02:00  460.0          456.0
2016-09-12 09:32:30+02:00  466.0          458.0
2016-09-12 09:32:31+02:00  467.0          460.0
2016-09-12 09:32:32+02:00  467.0          461.0
Out[36]:
ts
2016-09-12 09:32:22+02:00     12.0
2016-09-12 10:02:24+02:00    139.0
2016-09-12 16:56:11+02:00     17.0
2016-09-12 18:39:27+02:00    237.0
2016-09-12 19:17:36+02:00     11.0
2016-09-12 19:28:00+02:00    170.0
2016-09-12 19:46:23+02:00    203.0
2016-09-12 20:04:48+02:00     14.0
2016-09-12 20:36:55+02:00     12.0
2016-09-12 21:54:04+02:00     14.0
2016-09-12 22:24:35+02:00    274.0
Name: median_filter, dtype: float64

In [73]:
ax = data_homog.ldr.iloc[::5].plot(figsize=FS, lw=1, color='darkorange', alpha=.9)
data_homog['median_filter'].iloc[::5].plot(ax=ax, color='darkred', lw=2, alpha=.7)
for t in maximos.index:
    plt.plot(t, data_homog.loc[t, 'median_filter'], marker='o', color='red', markersize=5)
for t in minimos.index:
    plt.plot(t, data_homog.loc[t, 'median_filter'], marker='o', color='blue', markersize=5)

reconstr = data_homog['median_filter'].rolling(5, center=True).median()
for i, v in maximos.items():
    reconstr.loc[i:] -= v
for i, v in minimos.items():
    reconstr.loc[i:] -= v    
reconstr.rolling(300, center=True).median().plot(ax=ax, color='magenta', lw=6, alpha=.4)

ax.xaxis.set_major_locator(mpd.HourLocator())
ax.xaxis.set_major_formatter(mpd.DateFormatter('%H:%M'))



In [87]:
data_homog['pendiente_5min'] = data_homog['median_filter'].rolling(300, center=True).apply(lambda x: x[-1] - x[0]).fillna(0)
pend_dia = data_homog.loc[data_homog.sun, 'pendiente_5min']
pend_noche = data_homog.loc[~data_homog.sun, 'pendiente_5min']
r_dia = (pend_dia.max() - pend_dia.min()) // 2 
r_noche = (pend_noche.max() - pend_noche.min()) // 2 

ax = pend_dia.hist(bins=r_dia, figsize=FS, log=True, alpha=.6, lw=0, color=tableau20[6])
pend_noche.hist(bins=r_noche, ax=ax, log=True, alpha=.7, lw=0, color=tableau20[8])


/Users/uge/anaconda/envs/py35/lib/python3.5/site-packages/numpy/lib/function_base.py:564: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  n = np.zeros(bins, ntype)
/Users/uge/anaconda/envs/py35/lib/python3.5/site-packages/numpy/lib/function_base.py:611: VisibleDeprecationWarning: using a non-integer number instead of an integer will result in an error in the future
  n += np.bincount(indices, weights=tmp_w, minlength=bins).astype(ntype)
Out[87]:
<matplotlib.axes._subplots.AxesSubplot at 0x1262669b0>

In [91]:
sns.kdeplot(data_homog.loc[data_homog.sun, 'pendiente_5min'], data_homog.loc[data_homog.sun, 'median_filter'])


Out[91]:
<matplotlib.axes._subplots.AxesSubplot at 0x13780d278>

In [127]:
#sns.kdeplot(data_homog.loc[~data_homog.sun, 'pendiente_5min'].values, data_homog.loc[~data_homog.sun, 'median_filter'].values)
ax = data_homog.loc[data_homog.sun, 'disp_r7'][data_homog['disp_r7'] > 10].hist(bins=100, figsize=FS, lw=0, alpha=.6, color='red')
data_homog.loc[~data_homog.sun, 'disp_r7'][data_homog['disp_r7'] > 10].hist(bins=100, lw=0, alpha=.6, ax=ax)


Out[127]:
<matplotlib.axes._subplots.AxesSubplot at 0x142932da0>

In [ ]:


In [ ]:


In [170]:
dplot = data_homog.between_time('0:00', '23:59:59')

ax = dplot.drop(['poi', 'intervalo'], axis=1).iloc[::5].plot(figsize=FS)
ax.vlines(dplot[dplot['poi']].index, -600, 800, lw=1.5, color='y', alpha=.8)
(dplot['intervalo'] * 10).plot(ax=ax, lw=2, color='magenta', alpha=.8)
plt.grid('off')
ax.xaxis.set_major_locator(mpd.HourLocator())
ax.xaxis.set_major_formatter(mpd.DateFormatter('%H:%M'))
#data_homog.between_time('19:10', '20:00').plot(figsize=FS)
#data_homog.between_time('16:30', '17:30').plot(figsize=FS)
#data_homog['disp_r7'][data_homog['disp_r7'] > 20]



In [185]:
#print(dplot[dplot['poi']].head(7))
#dplot[(dplot.intervalo >= 0) & (dplot.intervalo <= 3)].plot(figsize=FS)
#dplot[dplot['poi']]

poi = dplot[dplot['poi']]
poi.set_index('intervalo').join(dplot.groupby('intervalo').median_filter.std().rename('std_mf')
                               ).join(dplot.groupby('intervalo').median_filter.mean().rename('mean_mf'))
#dplot.groupby('intervalo').median_filter.mean()


Out[185]:
ldr sun median_filter pendiente_5min disp_r7 disp_r5 poi pend_r5 pr_1 std_mf mean_mf
intervalo
1.0 439.0 True 455.0 48.0 13.0 4.0 True 4.0 True 16.214352 493.477440
2.0 325.0 True 335.0 -182.0 147.0 61.0 True -61.0 True 21.806663 348.375601
3.0 447.0 True 440.0 254.0 17.0 14.0 True 14.0 True 53.342194 473.470588
4.0 600.0 True 600.0 250.0 45.0 12.0 True 12.0 True 34.217051 709.213864
5.0 603.0 True 613.0 -106.0 46.0 17.0 True -17.0 True 42.199439 555.766391
6.0 492.0 True 493.0 -15.0 11.0 4.0 True -4.0 True 65.803933 551.726862
7.0 422.0 True 415.0 52.0 8.0 7.0 True 7.0 True 5.153110 427.250000
8.0 443.0 True 442.0 53.0 4.0 3.0 True 3.0 True 23.427459 393.452267
9.0 379.0 True 377.0 260.0 5.0 2.0 True 2.0 True 50.805855 391.519231
10.0 624.0 True 622.0 260.0 91.0 5.0 True 5.0 True 11.688663 611.932521
11.0 309.0 True 314.0 -303.0 132.0 3.0 True -3.0 True 1.787285 314.290323
12.0 337.0 True 328.0 17.0 4.0 2.0 True 2.0 True 9.230022 329.874799
13.0 610.0 True 534.0 294.0 193.0 153.0 True 153.0 True 17.717480 577.500000
14.0 609.0 True 603.0 294.0 14.0 7.0 True 7.0 True 26.419722 616.146154
15.0 340.0 True 340.0 -518.0 121.0 8.0 True -8.0 True 44.136333 313.875000
16.0 105.0 True 105.0 -518.0 84.0 8.0 True -8.0 True 12.516449 103.558594
17.0 328.0 True 312.0 220.0 43.0 8.0 True 8.0 True 6.689019 311.045956
18.0 290.0 True 290.0 -40.0 12.0 8.0 True -8.0 True 5.131601 284.000000
19.0 267.0 True 272.0 -40.0 8.0 5.0 True -5.0 True 2.138853 266.865942
20.0 287.0 True 283.0 20.0 4.0 3.0 True 3.0 True 14.053387 267.776480
21.0 274.0 False 263.0 27.0 4.0 2.0 True 2.0 True 1.648344 274.903829
22.0 256.0 False 256.0 -25.0 14.0 10.0 True -10.0 True 2.132489 252.489474
23.0 276.0 False 269.0 24.0 3.0 2.0 True 2.0 True 12.015578 279.731694
24.0 583.0 False 574.0 297.0 278.0 94.0 True 94.0 True 1.809163 576.422943

In [195]:
gb_int = dplot.tz_localize(None).reset_index().groupby('intervalo')
poi_data = gb_int.first().join(gb_int.median_filter.std().rename('std_mf')
                              ).join(gb_int.median_filter.median().rename('median_mf')
                                    ).join(gb_int.median_filter.apply(lambda x: np.mean(x[:10])).rename('first_mf')
                                          ).join(gb_int.median_filter.apply(lambda x: np.mean(x[-10:])).rename('last_mf'))
poi_data


Out[195]:
ts ldr sun median_filter pendiente_5min disp_r7 disp_r5 poi pend_r5 pr_1 std_mf median_mf first_mf last_mf
intervalo
0.0 2016-09-12 00:00:00 40.0 False 21.0 0.0 0.0 0.0 False 0.0 True 103.258718 31.0 30.1 440.50
1.0 2016-09-12 09:32:25 439.0 True 455.0 48.0 13.0 4.0 True 4.0 True 16.214352 495.0 458.1 478.30
2.0 2016-09-12 09:48:18 325.0 True 335.0 -182.0 147.0 61.0 True -61.0 True 21.806663 339.0 333.6 421.40
3.0 2016-09-12 10:02:10 447.0 True 440.0 254.0 17.0 14.0 True 14.0 True 53.342194 448.0 446.1 493.20
4.0 2016-09-12 10:02:27 600.0 True 600.0 250.0 45.0 12.0 True 12.0 True 34.217051 726.0 602.1 674.25
5.0 2016-09-12 11:24:19 603.0 True 613.0 -106.0 46.0 17.0 True -17.0 True 42.199439 546.0 609.9 503.20
6.0 2016-09-12 13:03:58 492.0 True 493.0 -15.0 11.0 4.0 True -4.0 True 65.803933 570.0 493.0 398.50
7.0 2016-09-12 16:56:16 422.0 True 415.0 52.0 8.0 7.0 True 7.0 True 5.153110 429.0 419.0 433.10
8.0 2016-09-12 16:57:12 443.0 True 442.0 53.0 4.0 3.0 True 3.0 True 23.427459 389.0 442.8 368.30
9.0 2016-09-12 18:38:38 379.0 True 377.0 260.0 5.0 2.0 True 2.0 True 50.805855 379.0 377.7 446.00
10.0 2016-09-12 18:39:30 624.0 True 622.0 260.0 91.0 5.0 True 5.0 True 11.688663 608.0 622.2 533.20
11.0 2016-09-12 19:15:04 309.0 True 314.0 -303.0 132.0 3.0 True -3.0 True 1.787285 314.0 314.0 318.50
12.0 2016-09-12 19:17:39 337.0 True 328.0 17.0 4.0 2.0 True 2.0 True 9.230022 330.0 329.1 369.90
13.0 2016-09-12 19:28:02 610.0 True 534.0 294.0 193.0 153.0 True 153.0 True 17.717480 575.0 573.1 582.60
14.0 2016-09-12 19:28:14 609.0 True 603.0 294.0 14.0 7.0 True 7.0 True 26.419722 621.0 606.9 544.90
15.0 2016-09-12 19:32:34 340.0 True 340.0 -518.0 121.0 8.0 True -8.0 True 44.136333 330.0 336.4 237.80
16.0 2016-09-12 19:33:38 105.0 True 105.0 -518.0 84.0 8.0 True -8.0 True 12.516449 103.0 104.5 167.90
17.0 2016-09-12 19:46:26 328.0 True 312.0 220.0 43.0 8.0 True 8.0 True 6.689019 308.5 315.1 300.60
18.0 2016-09-12 20:00:02 290.0 True 290.0 -40.0 12.0 8.0 True -8.0 True 5.131601 285.0 286.3 282.50
19.0 2016-09-12 20:00:15 267.0 True 272.0 -40.0 8.0 5.0 True -5.0 True 2.138853 266.0 271.5 271.60
20.0 2016-09-12 20:04:51 287.0 True 283.0 20.0 4.0 3.0 True 3.0 True 14.053387 270.0 283.2 253.60
21.0 2016-09-12 20:36:57 274.0 False 263.0 27.0 4.0 2.0 True 2.0 True 1.648344 275.0 263.8 270.60
22.0 2016-09-12 21:50:57 256.0 False 256.0 -25.0 14.0 10.0 True -10.0 True 2.132489 252.0 254.6 258.20
23.0 2016-09-12 21:54:07 276.0 False 269.0 24.0 3.0 2.0 True 2.0 True 12.015578 278.0 269.2 339.60
24.0 2016-09-12 22:24:37 583.0 False 574.0 297.0 278.0 94.0 True 94.0 True 1.809163 576.0 579.2 573.60

In [194]:
plt.scatter(x=poi_data.median_mf, y=poi_data.median_filter.)


Out[194]:
<matplotlib.collections.PathCollection at 0x151ffba58>

In [60]:
#irrad_dif_dir
pysolar.solar.get_altitude_fast?

In [64]:
idx_sun = data_homog[data_homog.sun].index
irrad_dif_dir = pd.concat([pd.Series(idx_sun, index=idx_sun).iloc[::50].apply(lambda x: pysolar.util.diffuse_underclear(LAT, LONG, x)
                                                  ).reindex(idx_sun).interpolate(),
                           pd.Series(idx_sun, index=idx_sun).iloc[::50].apply(lambda x: pysolar.radiation.get_radiation_direct(x, pysolar.solar.get_altitude_fast(LAT, LONG, x))
                                                  ).reindex(idx_sun).interpolate()], axis=1)

irrad_dif_dir.loc[::20].between_time('8:00', '20:00').plot(figsize=FS)


/Users/uge/anaconda/envs/py35/lib/python3.5/site-packages/pysolar/time.py:105: UserWarning: I don't know about leap seconds after 2015
  (leap_seconds_base_year + len(leap_seconds_adjustments) - 1)
Out[64]:
<matplotlib.axes._subplots.AxesSubplot at 0x1218fe668>

In [12]:
ldr = data_homog.ldr.values

@timeit('TOF', verbose=True)
def _TOF(arr):
    n = 0
    while n < 100:
        total_variation = np.sum(arr[1:] - arr[:-1])
        n += 1
    return total_variation


@timeit('TOF_ne', verbose=True)
def _TOF_ne(arr):
    n = 0
    while n < 100:
        #total_variation = ne.evaluate('sum()')
        a, b = arr[1:], arr[:-1]
        total_variation = ne.evaluate('sum(a - b)')
        n += 1
    return total_variation


_TOF(ldr)
_TOF_ne(ldr)


TOF TOOK: 0.008 s
TOF_ne TOOK: 0.016 s
Out[12]:
array(535.0)

In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [ ]:


In [3]:
data_solar.plot(figsize=(18, 8))
altura = data_solar.altitud[data_solar.altitud.abs() > 0]
azimut = data_solar.azimut[data_solar.altitud.abs() > 0]



In [ ]:


In [4]:
Image('http://www.monografias.com/trabajos82/energia-solar-fotovoltaica-y-sus-aplicaciones/image031.png')


Out[4]:

In [5]:
import math

def cosd(x):
    return np.cos(np.radians(x))

def sind(x):
    return np.sin(np.radians(x))

    
COS85_MIN = 0.0871557

def calc_cosenos(altitud_solar_deg, azimut_solar_deg, orientacion_N_deg, inclinacion_deg):
    cenit_deg = 90 - altitud_solar_deg
    azimut_efectivo = 180 + azimut_solar_deg - orientacion_N_deg

    # Ángulos de incidencia
    cos_theta = cosd(cenit_deg)*cosd(inclinacion_deg)+sind(cenit_deg)*sind(inclinacion_deg)*cosd(azimut_efectivo)
    cos_incidencia = cos_theta.apply(lambda x: max(0, x))
    cos_cenit_acotado = cosd(cenit_deg).apply(lambda x: max(COS85_MIN, x))
    return cos_incidencia, cos_cenit_acotado


orientacion_N_deg, inclinacion_deg = 90 - 27.5, 90
out = pd.concat(calc_cosenos(altura, azimut, orientacion_N_deg, inclinacion_deg), axis=1)

ax = out.plot(figsize=(18,10))


orientacion_N_deg, inclinacion_deg = - 27.5, 90
out_2 = pd.concat(calc_cosenos(altura, azimut, orientacion_N_deg, inclinacion_deg), axis=1)
out_2.plot(ax=ax)

(altura / altura.max()).plot(ax=ax, lw=3)


Out[5]:
<matplotlib.axes._subplots.AxesSubplot at 0x111677518>

In [ ]:


In [ ]:


In [11]:
COS85_MIN = 0.0871557

def _cos_incidencia(data, orientacion_N_deg, inclinacion_deg=90, 
                    alt='altitud', azi='azimut', irrad=None):  # 'irradiacion_cs'):
    cenit_deg = 90 - data[alt]
    azimut_efectivo = 180 + data[azi] - orientacion_N_deg

    # Ángulos de incidencia
    cos_theta = cosd(cenit_deg)*cosd(inclinacion_deg)+sind(cenit_deg)*sind(inclinacion_deg)*cosd(azimut_efectivo)
    cos_incidencia = cos_theta.apply(lambda x: max(0, x))
    
    if irrad is not None:
        if type(irrad) is str:
            return cos_incidencia * data[irrad]
        else:
            return cos_incidencia * irrad
    # cos_cenit_acotado = cosd(cenit_deg).apply(lambda x: max(COS85_MIN, x))
    return cos_incidencia

In [17]:
data_solar_p = data_solar.copy()
labels = ['irradiacion_cs']
for ang_N in range(-180, 180, 18):
    label = 'irrad_{}'.format(ang_N)
    data_solar_p[label] = _cos_incidencia(data_solar_p, ang_N, irrad='irradiacion_cs', inclinacion_deg=90)
    labels.append(label)
data_solar_p[labels].plot(figsize=(18, 12))


Out[17]:
<matplotlib.axes._subplots.AxesSubplot at 0x11399a668>

In [94]:
from numpy import pi as PI

i_CONSTANTE_SOLAR_E_0 = 1367  # W/m2
i_COS_OBLICUIDAD_ELIPTICA_RAD = .917477  # cosd(23.44º)
i_SIN_OBLICUIDAD_ELIPTICA_RAD = .397789  # sind(23.44º)

i_DURACION_ANYO_SOLAR = 365.24  # W/m2
i_DIAS_DESDE_SOLST_INV_A_DIA_1 = 10.
i_DIAS_DESDE_DIA_1_A_PERIHELIO = 2.

i_SIGMA_STEFAN_BOLTZMANN = 5.67e-8  # W/(m^2·K^4)

i_VALOR_FORM_FACT_CENIT_PEREZ_K_RAD = 1.041
i_VALOR_COS_85_MINIMO = 0.0871557
i_DEG_A_RAD = PI / 180.

i_EMISIVIDAD_OPACOS_CTE = .9
i_RESISTENCIA_SUPERFICIAL_EXTERIOR = .04
i_PREC_COMPARA = .01


def get_julian_day_of_year(when):
    jd = when.dayofyear
    return jd + when.hour / 24. + when.minute / (24. * 60) + when.second / (24. * 3600)
    
    
def i_calcula_desviacion_eq_of_time_y_declinacion(dia_juliano):
    ang_anual_unit_rad = 2. * PI / i_DURACION_ANYO_SOLAR
    A = ang_anual_unit_rad * (dia_juliano + i_DIAS_DESDE_SOLST_INV_A_DIA_1)
    B = A + 2. * .0167 * np.sin(ang_anual_unit_rad * (dia_juliano - i_DIAS_DESDE_DIA_1_A_PERIHELIO))
    sin_declinacion = -i_SIN_OBLICUIDAD_ELIPTICA_RAD * np.cos(B)
    cos_declinacion = np.cos(np.arcsin(sin_declinacion))
    C = (A - np.arctan(np.tan(B) / i_COS_OBLICUIDAD_ELIPTICA_RAD)) / PI
    EoT_opc = 720. * (C - np.floor(C + 0.5 + 0.0000001))
    return sin_declinacion, cos_declinacion, EoT_opc



# Ecuation of time
def i_calcula_declinacion(dia_juliano):
    ang_anual_unit_rad = 2. * PI / i_DURACION_ANYO_SOLAR
    A = ang_anual_unit_rad * (dia_juliano + i_DIAS_DESDE_SOLST_INV_A_DIA_1)
    B = A + 2. * .0167 * np.sin(ang_anual_unit_rad * (dia_juliano - i_DIAS_DESDE_DIA_1_A_PERIHELIO))
    sin_declinacion = -i_SIN_OBLICUIDAD_ELIPTICA_RAD * np.cos(B)
    cos_declinacion = np.cos(np.arcsin(sin_declinacion))
    return sin_declinacion, cos_declinacion


def enhsol_calcula_altura_azimut_en_hora_solar(f_dia_anual, latitud_rad, con_correccion_eot):

    def i_angulo_horario_de_f_dia_anual(f_dia):
        hora_diaria = f_dia % 24. * 24.
        return PI * (hora_diaria / 12. - 1.)

    if con_correccion_eot:
        sin_declinacion, cos_declinacion, correccion_EoT_min = i_calcula_desviacion_eq_of_time_y_declinacion(f_dia_anual)
    else:
        correccion_EoT_min = 0
        sin_declinacion, cos_declinacion = i_calcula_declinacion(f_dia_anual)

    angulo_horario = i_angulo_horario_de_f_dia_anual(f_dia_anual)
    sin_alfa_s = np.sin(latitud_rad) * sin_declinacion + np.cos(latitud_rad) * cos_declinacion * np.cos(angulo_horario)
    cos_gamma_s = (sin_declinacion - sin_alfa_s * np.sin(latitud_rad)) / (np.cos(latitud_rad) * np.sqrt(1 - sin_alfa_s**2))

    altura = np.arcsin(sin_alfa_s) / i_DEG_A_RAD

    if altura > 0.:
        azimut = np.arccos(cos_gamma_s) / i_DEG_A_RAD

        if (angulo_horario < 0.):
            azimut = azimut - 180.
        else:
            azimut = 180. - azimut

        if (con_correccion_eot == True):
            azimut += correccion_EoT_min / 4.
    else:
        altura = 0.
        azimut = 0.

    return altura, azimut, 90. - altura  # cenit_solar_deg


get_julian_day_of_year(ts)
print_red(i_calcula_desviacion_eq_of_time_y_declinacion(get_julian_day_of_year(ts)))

con_correccion_eot = True
enhsol_calcula_altura_azimut_en_hora_solar(get_julian_day_of_year(pd.Timestamp.now()), LAT * i_DEG_A_RAD, con_correccion_eot)


(0.16903286917637006, 0.98561041448333131, -1.1440401066674877)
Out[94]:
(0.0, 0.0, 90.0)

In [34]:
ts = pd.Timestamp(day)
pysolar.solar.time.get_julian_solar_day(ts) % 365


/Users/uge/anaconda/envs/py35/lib/python3.5/site-packages/pysolar/time.py:105: UserWarning: I don't know about leap seconds after 2015
  (leap_seconds_base_year + len(leap_seconds_adjustments) - 1)
Out[34]:
82.41667586797848

In [44]:
pysolar.solar.equation_of_time(get_julian_day_of_year(ts))


240 2016-08-27 00:00:00
Out[44]:
-0.6748056688866866

In [60]:
compara_eot = pd.DataFrame([(i_calcula_desviacion_eq_of_time_y_declinacion(t)[2], 
                             pysolar.solar.equation_of_time(t)) 
                            for t in range(365)], columns=['cype', 'pysolar'])
compara_eot.plot(figsize=(18, 10))


Out[60]:
<matplotlib.axes._subplots.AxesSubplot at 0x113c72438>

In [100]:
jd_arr = data_solar.index.dayofyear + data_solar.index.hour / 24. + data_solar.index.minute / (24. * 60) + data_solar.index.second / (24. * 3600)

compara_alt_azi = pd.DataFrame([(*enhsol_calcula_altura_azimut_en_hora_solar(jd, 
                                                                             LAT * i_DEG_A_RAD, con_correccion_eot)[:2],
                                 pysolar.solar.get_altitude(LAT, LONG, t), 
                                 pysolar.solar.get_azimuth(LAT, LONG, t)) for t, jd in zip(data_solar.index, jd_arr)], 
                               columns=['alt_cype', 'azi_cype', 'alt', 'azi'], index=data_solar.index)
compara_alt_azi.plot(figsize=(18, 12))


/Users/uge/anaconda/envs/py35/lib/python3.5/site-packages/pysolar/time.py:105: UserWarning: I don't know about leap seconds after 2015
  (leap_seconds_base_year + len(leap_seconds_adjustments) - 1)
Out[100]:
<matplotlib.axes._subplots.AxesSubplot at 0x1123c7438>

In [101]:
compara_alt_azi.apply(np.argmax)


Out[101]:
alt_cype   2016-08-27 12:00:00+02:00
azi_cype   2016-08-27 18:30:00+02:00
alt        2016-08-27 14:04:00+02:00
azi        2016-08-27 14:06:00+02:00
dtype: datetime64[ns, Europe/Madrid]

In [102]:
t = 240
i_calcula_desviacion_eq_of_time_y_declinacion(t)[2], pysolar.solar.equation_of_time(t)


Out[102]:
(-1.1440401066674877, -0.6748056688866866)

In [108]:
LONG / 360 * 24 * 60


Out[108]:
-3.4656080000000005

In [128]:
t


Out[128]:
240

In [119]:
# Prepara datos para comparación "fina":
#solar.iloc[::10].plot()

compara = solar[(solar.artif_level_max == 0) & (solar.altitud > 0)][['ldr', 'altitud', 'azimut', 'irradiacion_cs']].copy()
compara.plot()


Out[119]:
<matplotlib.axes._subplots.AxesSubplot at 0x111307cf8>

In [130]:
compara.altitud.argmax(), pysolar.solar.equation_of_time(pd.Timestamp('2016-08-27 14:05:06', tz=TZ).dayofyear)


Out[130]:
(Timestamp('2016-08-27 14:05:06+0200', tz='Europe/Madrid', offset='2S'),
 -0.6748056688866866)

In [131]:
otro = get_solar_day(day=pd.Timestamp('2016-02-18'), step_minutes=1, tz=TZ)
otro.altitud.argmax(), pysolar.solar.equation_of_time(pd.Timestamp('2016-02-18 14:05:06', tz=TZ).dayofyear)


Out[131]:
(Timestamp('2016-02-18 13:18:00+0100', tz='Europe/Madrid'),
 -14.440435531895162)

In [133]:
long_minutes = LONG / 360 * (24 * 60)
long_minutes


Out[133]:
-3.465608

In [135]:
(pysolar.solar.equation_of_time(pd.Timestamp('2016-02-18 14:05:06', tz=TZ).dayofyear), 
pysolar.solar.equation_of_time(get_julian_day_of_year(pd.Timestamp('2016-02-18 14:05:06', tz=TZ))), 
pysolar.solar.equation_of_time(pd.Timestamp('2016-02-19 14:05:06', tz=TZ).dayofyear))


Out[135]:
(-14.440435531895162, -14.401441583349268, -14.37149278895695)

In [246]:
data_solar_p = compara.iloc[::100].copy()
labels = []
for ang_N in range(-180, 180, 5):
    label = 'irrad_{}'.format(ang_N)
    data_solar_p[label] = _cos_incidencia(data_solar_p, ang_N, irrad='irradiacion_cs', inclinacion_deg=90)
    labels.append(label)

ax = data_solar_p[['ldr', 'irradiacion_cs']].plot(figsize=(18, 12), lw=3)
for c in labels:
    data_solar_p[c].plot(ax=ax, lw=.75, legend=c)



In [150]:
data_solar_p = compara.iloc[::100].copy()
labels = []
for ang_N in range(-180, 180, 30):
    label = 'irrad_{}'.format(ang_N)
    data_solar_p[label] = _cos_incidencia(data_solar_p, ang_N, irrad='irradiacion_cs', inclinacion_deg=10)
    labels.append(label)

ax = data_solar_p[['ldr', 'irradiacion_cs']].plot(figsize=(18, 12), lw=3)
for c in labels:
    data_solar_p[c].plot(ax=ax, lw=.75, legend=c)



In [214]:
data_solar_p = compara.iloc[::100].copy()
labels = []
for ang_N in range(-180, 180, 5):
    label = 'irrad_{}'.format(ang_N)
    data_solar_p[label] = _cos_incidencia(data_solar_p, ang_N, irrad='irradiacion_cs', inclinacion_deg=45)
    labels.append(label)

ax = data_solar_p[['ldr', 'irradiacion_cs']].plot(figsize=(18, 12), lw=3)
for c in labels:
    data_solar_p[c].plot(ax=ax, lw=.75, legend=c)



In [247]:
import statsmodels.api as sm

X, Y = data_solar_p['ldr'], data_solar_p.drop(['altitud', 'azimut', 'ldr'], axis=1)
model = sm.OLS(X, Y).fit()
model.summary()

(model.params * Y).sum(axis=1).plot(figsize=(18, 10), color='darkred')
X.plot(lw=3, alpha=.7, color='darkgreen')

model.pvalues[model.pvalues > .5]


Out[247]:
irrad_-100    0.740967
irrad_80      0.831274
dtype: float64

In [248]:
(model.params[model.params > 0] * Y[model.params[model.params > 0].index]).sum(axis=1).plot()


Out[248]:
<matplotlib.axes._subplots.AxesSubplot at 0x133383908>

In [278]:
from enerpi.base import timeit


@timeit('separa_ldr_artificial_natural', verbose=True)
def _separa_ldr_artificial_natural(data_day, resample_inicial='2s', delta_roll_threshold=100):

    def _f_roll(x):
        delta_pos = np.sum(x[x > 0])
        delta_neg = np.sum(x[x < 0])
        return delta_pos + delta_neg

    def _last_on(x):
        if (np.sum(x) == 1.) & (x[-1] == 1.):
            return True
        return False

    # TODO Resolver NaN's
    data_analysis = pd.DataFrame(data_day['ldr'].resample(resample_inicial).mean().interpolate().dropna())

    data_analysis['delta_roll'] = data_analysis.ldr.diff().fillna(0).rolling(3, center=True).apply(_f_roll).fillna(0)

    data_analysis['ch_state'] = 0
    data_analysis.loc[data_analysis[(data_analysis['delta_roll'] > delta_roll_threshold
                                     ).rolling(3).apply(_last_on).fillna(False) > 0].index, 'ch_state'] = 1
    data_analysis.loc[data_analysis[(data_analysis['delta_roll'] < -delta_roll_threshold
                                     ).rolling(3).apply(_last_on).fillna(False) > 0].index, 'ch_state'] = -1

    cambios = data_analysis[data_analysis['ch_state'] != 0]
    print_red(cambios)

    data_analysis['artif_level_max'] = 0
    apagados = data_analysis[data_analysis['ch_state'] == -1].index
    for i_enc in data_analysis[data_analysis['ch_state'] == 1].index:
        apagado = apagados[apagados > i_enc]
        if len(apagado) > 0:
            apagado = apagado[0]
            data_analysis.loc[i_enc:apagado, 'artif_level_max'] = data_analysis.ldr.loc[i_enc:apagado].max()
            print_red('Encendido L={:.0f}, SEGS={:.0f}'.format(data_analysis.ldr.loc[i_enc:apagado].max(),
                                                               (apagado - i_enc).total_seconds()))

    # Reconstrucción LDR natural:
    hay_artificial = data_analysis.artif_level_max > 0
    index_art = data_analysis[hay_artificial.shift(-2) | hay_artificial.shift(2)].index

    rs_natural = data_analysis.ldr.drop(index_art).resample('2min')
    data_analysis_simple = pd.concat([rs_natural.max().rename('nat_max').interpolate(),
                                      rs_natural.min().rename('nat_min').interpolate(),
                                      rs_natural.mean().rename('nat_mean').interpolate()], axis=1)
    # data_analysis_simple = data_analysis_simple.join(data_analysis[['altitud', 'azimut']])
    return data_analysis, data_analysis_simple



day = '2016-09-09'
d_sample = LDR.loc[day:day]
d_sample, d_sample_simple = separa_ldr_artificial_natural(d_sample, 
                                                          resample_inicial='2s', delta_roll_threshold=100)
d_sample.iloc[::100].plot()


                             ldr  delta_roll  ch_state
ts                                                    
2016-09-09 11:46:12+02:00  409.5       102.0         1
2016-09-09 20:35:16+02:00   28.0       242.5         1
2016-09-09 21:26:40+02:00  524.0      -394.0        -1
2016-09-09 22:06:44+02:00   68.0       256.5         1
2016-09-09 22:20:12+02:00  515.0      -414.5        -1
Encendido L=594, SEGS=34828
Encendido L=594, SEGS=3084
Encendido L=581, SEGS=808
                            ldr  delta_roll  ch_state  artif_level_max
ts                                                                    
2016-09-09 00:00:00+02:00  37.0         0.0         0              0.0
2016-09-09 00:00:02+02:00  37.0         0.0         0              0.0
2016-09-09 00:00:04+02:00  37.0         0.5         0              0.0
2016-09-09 00:00:06+02:00  37.5        -0.5         0              0.0
2016-09-09 00:00:08+02:00  36.5         0.5         0              0.0
                            ldr  delta_roll  ch_state  artif_level_max
ts                                                                    
2016-09-09 00:00:00+02:00  37.0         0.0         0              0.0
2016-09-09 00:00:02+02:00  37.0         0.0         0              0.0
2016-09-09 00:00:04+02:00  37.0         0.5         0              0.0
2016-09-09 00:00:06+02:00  37.5        -0.5         0              0.0
2016-09-09 00:00:08+02:00  36.5         0.5         0              0.0
Out[278]:
<matplotlib.axes._subplots.AxesSubplot at 0x13794db70>

In [279]:
d_sample.iloc[::100].delta_roll.plot()


Out[279]:
<matplotlib.axes._subplots.AxesSubplot at 0x1378902b0>

In [304]:
def _roll_band(x):
    ab = np.max(x) - np.min(x)
    
    #if ab > 15:
    #    return ab
    #return 0
    return ab / np.mean(x)

d_sample.ldr.rolling(15, center=True).apply(_roll_band).plot()


Out[304]:
<matplotlib.axes._subplots.AxesSubplot at 0x1302b1278>

In [305]:
#((d_sample.ldr).rolling(15).mean().pct_change()).plot()
(d_sample.ldr.rolling(15, center=True).apply(_roll_band) * d_sample.ldr).plot()


Out[305]:
<matplotlib.axes._subplots.AxesSubplot at 0x144f155c0>

In [297]:
((d_sample.ldr).rolling(15).mean().diff()).plot()


Out[297]:
<matplotlib.axes._subplots.AxesSubplot at 0x12aaf8828>

In [222]:
cols_pos = model.params[model.params > 0].index

model_p = sm.OLS(X, Y[cols_pos]).fit()
model_p.summary()


Out[222]:
OLS Regression Results
Dep. Variable: ldr R-squared: 1.000
Model: OLS Adj. R-squared: 0.999
Method: Least Squares F-statistic: 8496.
Date: mié, 14 sep 2016 Prob (F-statistic): 6.12e-231
Time: 11:39:49 Log-Likelihood: -720.57
No. Observations: 188 AIC: 1519.
Df Residuals: 149 BIC: 1645.
Df Model: 39
Covariance Type: nonrobust
coef std err t P>|t| [95.0% Conf. Int.]
irrad_-180 0.3508 0.885 0.396 0.692 -1.398 2.099
irrad_-175 0.5943 0.687 0.865 0.388 -0.764 1.952
irrad_-160 -3.2775 0.269 -12.165 0.000 -3.810 -2.745
irrad_-155 1.0585 0.338 3.129 0.002 0.390 1.727
irrad_-150 -0.1641 0.363 -0.453 0.652 -0.881 0.552
irrad_-145 0.0733 0.317 0.232 0.817 -0.552 0.699
irrad_-120 -0.4760 0.360 -1.324 0.188 -1.186 0.234
irrad_-115 0.5546 0.580 0.956 0.341 -0.592 1.701
irrad_-110 1.4974 0.620 2.413 0.017 0.271 2.723
irrad_-95 -0.0354 0.803 -0.044 0.965 -1.622 1.551
irrad_-90 -0.0495 0.671 -0.074 0.941 -1.376 1.277
irrad_-80 -0.0660 0.605 -0.109 0.913 -1.262 1.130
irrad_-50 0.0307 0.558 0.055 0.956 -1.072 1.133
irrad_-45 -0.3586 0.750 -0.478 0.633 -1.840 1.123
irrad_-40 -0.1818 0.937 -0.194 0.846 -2.034 1.670
irrad_-35 -0.1613 0.471 -0.342 0.732 -1.092 0.769
irrad_-30 -0.4617 1.119 -0.413 0.680 -2.673 1.749
irrad_-25 -0.5626 0.480 -1.172 0.243 -1.511 0.386
irrad_-20 0.4897 0.413 1.186 0.238 -0.326 1.306
irrad_-15 0.2399 2.017 0.119 0.905 -3.745 4.225
irrad_-10 289.2305 34.377 8.414 0.000 221.301 357.160
irrad_0 0.1961 1.102 0.178 0.859 -1.981 2.374
irrad_5 1.0211 0.553 1.847 0.067 -0.072 2.114
irrad_10 1.2755 0.278 4.580 0.000 0.725 1.826
irrad_35 0.5091 0.347 1.466 0.145 -0.177 1.195
irrad_40 -0.6754 0.296 -2.282 0.024 -1.260 -0.091
irrad_65 -0.1140 0.698 -0.163 0.870 -1.493 1.265
irrad_70 0.0604 0.800 0.075 0.940 -1.521 1.641
irrad_85 0.0628 0.614 0.102 0.919 -1.151 1.277
irrad_95 -0.0752 0.628 -0.120 0.905 -1.315 1.165
irrad_100 0.0013 0.719 0.002 0.999 -1.420 1.423
irrad_105 0.0985 0.706 0.140 0.889 -1.296 1.493
irrad_110 -0.3081 0.656 -0.469 0.639 -1.605 0.989
irrad_115 0.3567 0.556 0.641 0.522 -0.743 1.456
irrad_130 0.1539 1.008 0.153 0.879 -1.838 2.146
irrad_145 0.6849 1.024 0.669 0.504 -1.338 2.708
irrad_160 0.3538 1.614 0.219 0.827 -2.836 3.543
irrad_170 -5.7508 2.310 -2.490 0.014 -10.315 -1.187
irrad_175 6.5261 1.021 6.394 0.000 4.509 8.543
Omnibus: 118.424 Durbin-Watson: 0.494
Prob(Omnibus): 0.000 Jarque-Bera (JB): 965.731
Skew: 2.289 Prob(JB): 1.97e-210
Kurtosis: 13.116 Cond. No. 5.05e+04

In [229]:
model_p.pvalues[model_p.pvalues > .5]


Out[229]:
irrad_-180    0.692364
irrad_-150    0.651530
irrad_-145    0.817110
irrad_-95     0.964855
irrad_-90     0.941273
irrad_-80     0.913331
irrad_-50     0.956235
irrad_-45     0.633042
irrad_-40     0.846476
irrad_-35     0.732486
irrad_-30     0.680495
irrad_-15     0.905486
irrad_0       0.859033
irrad_65      0.870447
irrad_70      0.939939
irrad_85      0.918709
irrad_95      0.904772
irrad_100     0.998529
irrad_105     0.889200
irrad_110     0.639466
irrad_115     0.522353
irrad_130     0.878868
irrad_145     0.504419
irrad_160     0.826820
dtype: float64

In [221]:
(model_p.params * Y[cols_pos]).sum(axis=1).plot()
X.plot()


Out[221]:
<matplotlib.axes._subplots.AxesSubplot at 0x1375379e8>

In [172]:
dir(scipy.optimize)

scipy.optimize.curve_fit?

In [220]:
sol_v, rnorm = scipy.optimize.nnls(Y, X)
ax = (sol_v * Y).sum(axis=1).plot()
(sol_v * Y).plot(ax=ax)
X.plot()


Out[220]:
<matplotlib.axes._subplots.AxesSubplot at 0x12ab2cac8>

In [197]:
def local_maxima_minima(serie, window_delta='2h', verbose=True):

    def _is_local_min(x):
        if np.argmin(x) == len(x) // 2:
            return True
        return False

    def _is_local_max(x):
        if np.argmax(x) == len(x) // 2:
            return True
        return False

    roll_w = int(pd.Timedelta(window_delta) / serie.index.freq)
    if roll_w % 2 == 0:
        roll_w += 1
    if verbose:
        print_secc('LOCAL MAX-MIN de {} valores. Centered window of {} samples (for ∆T={}, freq={})'
                   .format(len(X), roll_w, window_delta, serie.index.freq))
    maximos = serie.rolling(roll_w, center=True).apply(_is_local_max).fillna(0).astype(bool)
    minimos = serie.rolling(roll_w, center=True).apply(_is_local_min).fillna(0).astype(bool)
    if verbose:
        print_info(serie[maximos])
        print_cyan(serie[minimos])
    return maximos, minimos

maximos, minimos = local_maxima_minima(X, window_delta='2h', verbose=True)

ax = X.plot(figsize=(18,10))
X[maximos].plot(ax=ax, lw=0, marker='o', color='red')
X[minimos].plot(ax=ax, lw=0, marker='o', color='blue')
#ax.set_xlim((X[maximos].index[0], X[maximos].index[-1]))


 ==> LOCAL MAX-MIN de 188 valores. Centered window of 37 samples (for ∆T=2h, freq=<200 * Seconds>)
ts
2016-08-27 10:36:32+02:00    676.0
2016-08-27 15:23:12+02:00    651.0
Name: ldr, dtype: float64
ts
2016-08-27 13:09:52+02:00    465.5
Freq: 200S, Name: ldr, dtype: float64
Out[197]:
<matplotlib.axes._subplots.AxesSubplot at 0x127ffe518>

In [ ]:


In [ ]:


In [ ]: